Merge pull request #551 from wtt40122/master

es quey update
XiaoMi · Jul 14, 2023 · 6be8c7f · 6be8c7f
2 parents ad7d3ae + 6f81b78
commit 6be8c7f
Show file tree

Hide file tree

Showing 9 changed files with 98 additions and 30 deletions.
diff --git a/hera-all/hera-log/log-manager/pom.xml b/hera-all/hera-log/log-manager/pom.xml
@@ -36,10 +36,22 @@
             <version>5.3.25</version>
         </dependency>
 
+        <dependency>
+            <groupId>run.mone</groupId>
+            <artifactId>spring-context-support</artifactId>
+            <version>1.0.10-mone-SNAPSHOT</version>
+        </dependency>
+
         <dependency>
             <groupId>run.mone</groupId>
             <artifactId>dubbo</artifactId>
             <version>2.7.12-mone-v8-SNAPSHOT</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.alibaba.spring</groupId>
+                    <artifactId>spring-context-support</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>
@@ -102,10 +114,23 @@
             </exclusions>
         </dependency>
 
+        <dependency>
+            <groupId>run.mone</groupId>
+            <artifactId>es</artifactId>
+            <version>1.5-SNAPSHOT</version>
+            <scope>compile</scope>
+        </dependency>
+
         <dependency>
             <groupId>run.mone</groupId>
             <artifactId>docean-plugin-es</artifactId>
             <version>1.4-SNAPSHOT</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>run.mone</groupId>
+                    <artifactId>es</artifactId>
+                </exclusion>
+            </exclusions>
         </dependency>
 
         <dependency>

diff --git a/...all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java b/...all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java
@@ -29,7 +29,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.aggregations.AggregationBuilder;

diff --git a/...-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java b/...-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java
@@ -17,6 +17,8 @@
 
 import com.xiaomi.mone.log.manager.model.vo.LogContextQuery;
 import com.xiaomi.mone.log.manager.model.vo.LogQuery;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory;
 import com.xiaomi.mone.log.manager.service.statement.StatementMatchParseFactory;
 import com.xiaomi.youpin.docean.anno.Service;
 import com.xiaomi.youpin.docean.common.DoceanConfig;
@@ -38,6 +40,13 @@
 @Slf4j
 @Service
 public class SearchLog {
+
+    private CommonExtensionService commonExtensionService;
+
+    public void init() {
+        commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService();
+    }
+
     /**
      * 获取查询参数
      *
@@ -58,16 +67,14 @@ public BoolQueryBuilder getQueryBuilder(LogQuery logQuery, List<String> keyList)
         return boolQueryBuilder;
     }
 
-    private static BoolQueryBuilder buildCommonBuilder(LogQuery logQuery) {
-        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
-        boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(logQuery.getStartTime()).to(logQuery.getEndTime()));
-        boolQueryBuilder.filter(QueryBuilders.termQuery("logstore", logQuery.getLogstore()));
+    private BoolQueryBuilder buildCommonBuilder(LogQuery logQuery) {
+        BoolQueryBuilder boolQueryBuilder = commonExtensionService.commonRangeQuery(logQuery);
         // 支持tail多选
         if (StringUtils.isNotEmpty(logQuery.getTail())) {
             BoolQueryBuilder tailQueryBuilder = QueryBuilders.boolQuery();
             String[] tailLimitArray = logQuery.getTail().split(",");
             for (String tail : tailLimitArray) {
-                tailQueryBuilder.should(QueryBuilders.termQuery("tail", tail));
+                tailQueryBuilder.should(commonExtensionService.multipleChooseBuilder(logQuery.getStoreId(), tail));
             }
             tailQueryBuilder.minimumShouldMatch(1);
             boolQueryBuilder.filter(tailQueryBuilder);
@@ -283,14 +290,7 @@ private static int getEndIndex(String querytext) {
     }
 
     public boolean isLegalParam(LogContextQuery param) {
-        if (param == null
-                || StringUtils.isEmpty(param.getLogstore())
-                || StringUtils.isEmpty(param.getIp())
-                || StringUtils.isEmpty(param.getFileName())
-                || param.getLineNumber() == null
-                || StringUtils.isEmpty(param.getTimestamp())
-                || param.getType() == null
-                || param.getPageSize() == null) {
+        if (param == null || StringUtils.isEmpty(param.getLogstore()) || StringUtils.isEmpty(param.getIp()) || StringUtils.isEmpty(param.getFileName()) || param.getLineNumber() == null || StringUtils.isEmpty(param.getTimestamp()) || param.getType() == null || param.getPageSize() == null) {
             return false;
         }
         return true;

diff --git a/...ain/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java b/...ain/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java
@@ -15,6 +15,10 @@
  */
 package com.xiaomi.mone.log.manager.service.extension.common;
 
+import com.xiaomi.mone.log.manager.model.vo.LogQuery;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.TermQueryBuilder;
+
 /**
  * @author wtt
  * @version 1.0
@@ -33,4 +37,10 @@ public interface CommonExtensionService {
     String getMachineRoomName(String machineRoomEn);
 
     boolean middlewareEnumValid(Integer type);
+
+    BoolQueryBuilder commonRangeQuery(LogQuery logQuery);
+
+    TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal);
+
+    String queryDateHistogramField(Long storeId);
 }
diff --git a/...a/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java b/...a/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java
@@ -17,8 +17,12 @@
 
 import com.xiaomi.mone.log.api.enums.MachineRegionEnum;
 import com.xiaomi.mone.log.api.enums.MiddlewareEnum;
+import com.xiaomi.mone.log.manager.model.vo.LogQuery;
 import com.xiaomi.youpin.docean.anno.Service;
 import lombok.extern.slf4j.Slf4j;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
 
 import static com.xiaomi.mone.log.common.Constant.DEFAULT_STREAM_SERVER_NAME;
 import static com.xiaomi.mone.log.common.Constant.LOG_MANAGE_PREFIX;
@@ -53,4 +57,22 @@ public String getMachineRoomName(String machineRoomEn) {
     public boolean middlewareEnumValid(Integer type) {
         return MiddlewareEnum.ROCKETMQ.getCode().equals(type);
     }
+
+    @Override
+    public BoolQueryBuilder commonRangeQuery(LogQuery logQuery) {
+        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+        boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(logQuery.getStartTime()).to(logQuery.getEndTime()));
+        boolQueryBuilder.filter(QueryBuilders.termQuery("logstore", logQuery.getLogstore()));
+        return boolQueryBuilder;
+    }
+
+    @Override
+    public TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal) {
+        return QueryBuilders.termQuery("tail", chooseVal);
+    }
+
+    @Override
+    public String queryDateHistogramField(Long storeId) {
+        return "timestamp";
+    }
 }
diff --git a/...log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java b/...log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java
@@ -21,7 +21,6 @@
 import com.xiaomi.mone.log.api.model.dto.TraceLogDTO;
 import com.xiaomi.mone.log.api.model.vo.TraceLogQuery;
 import com.xiaomi.mone.log.api.service.LogDataService;
-import com.xiaomi.mone.log.common.HeraLocalCache;
 import com.xiaomi.mone.log.common.Result;
 import com.xiaomi.mone.log.exception.CommonError;
 import com.xiaomi.mone.log.manager.common.context.MoneUserContext;
@@ -44,6 +43,8 @@
 import com.xiaomi.mone.log.manager.model.vo.TraceAppLogUrlQuery;
 import com.xiaomi.mone.log.manager.service.EsDataBaseService;
 import com.xiaomi.mone.log.manager.service.EsDataService;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory;
 import com.xiaomi.mone.log.parse.LogParser;
 import com.xiaomi.youpin.docean.anno.Service;
 import com.xiaomi.youpin.docean.common.StringUtils;
@@ -55,8 +56,7 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.client.core.CountRequest;
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;
@@ -106,6 +106,12 @@ public class EsDataServiceImpl implements EsDataService, LogDataService, EsDataB
     @Reference(interfaceClass = LogDataService.class, group = "$dubbo.youpin.group", check = false, timeout = 5000)
     private LogDataService logDataService;
 
+    private CommonExtensionService commonExtensionService;
+
+    public void init() {
+        commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService();
+    }
+
     private Set<String> noHighLightSet = new HashSet<>();
 
     private Set<String> hidenFiledSet = new HashSet<>();
@@ -247,7 +253,7 @@ public Result<EsStatisticResult> EsStatistic(LogQuery logQuery) throws Exception
         try {
             EsStatisticResult result = new EsStatisticResult();
             result.setName(constractEsStatisticRet(logQuery));
-            MilogLogStoreDO logStore = logstoreDao.getByName(logQuery.getLogstore());
+            MilogLogStoreDO logStore = logstoreDao.queryById(logQuery.getStoreId());
             if (logStore == null) {
                 return new Result<>(CommonError.UnknownError.getCode(), "not found logstore", null);
             }
@@ -260,7 +266,8 @@ public Result<EsStatisticResult> EsStatistic(LogQuery logQuery) throws Exception
             }
             if (!StringUtils.isEmpty(interval)) {
                 BoolQueryBuilder queryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStore.getKeyList()));
-                EsClient.EsRet esRet = esService.dateHistogram(esIndex, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder);
+                String histogramField = commonExtensionService.queryDateHistogramField(logQuery.getStoreId());
+                EsClient.EsRet esRet = esService.dateHistogram(esIndex, histogramField, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder);
                 result.setCounts(esRet.getCounts());
                 result.setTimestamps(esRet.getTimestamps());
                 result.setQueryBuilder(queryBuilder);
@@ -448,11 +455,7 @@ public void logExport(LogQuery logQuery) throws Exception {
         int maxLogNum = 10000;
         logQuery.setPageSize(maxLogNum);
         Result<LogDTO> logDTOResult = this.logQuery(logQuery);
-        List<Map<String, Object>> exportData =
-                logDTOResult.getCode() != CommonError.Success.getCode()
-                        || logDTOResult.getData().getLogDataDTOList() == null
-                        || logDTOResult.getData().getLogDataDTOList().isEmpty() ?
-                        null : logDTOResult.getData().getLogDataDTOList().stream().map(logDataDto -> ExportUtils.SplitTooLongContent(logDataDto)).collect(Collectors.toList());
+        List<Map<String, Object>> exportData = logDTOResult.getCode() != CommonError.Success.getCode() || logDTOResult.getData().getLogDataDTOList() == null || logDTOResult.getData().getLogDataDTOList().isEmpty() ? null : logDTOResult.getData().getLogDataDTOList().stream().map(logDataDto -> ExportUtils.SplitTooLongContent(logDataDto)).collect(Collectors.toList());
         HSSFWorkbook excel = ExportExcel.HSSFWorkbook4Map(exportData, generateTitle(logQuery));
         // 下载
         String fileName = String.format("%s_log.xls", logQuery.getLogstore());

diff --git a/...g-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java b/...g-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java
@@ -47,7 +47,7 @@
 import org.elasticsearch.ElasticsearchStatusException;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.core.TimeValue;
+import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.search.SearchHit;

diff --git a/...og-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java b/...og-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java
@@ -469,23 +469,27 @@ public Result<Void> deleteLogTail(Long id) {
         if (StringUtils.isNotEmpty(validMsg)) {
             return new Result<>(CommonError.ParamsError.getCode(), validMsg);
         }
+
+        deleteRemoteConfig(id, milogLogtailDo);
+
         if (milogLogtailDao.deleteMilogLogtail(id)) {
             MilogLogStoreDO logStoreDO = logStoreDao.queryById(milogLogtailDo.getStoreId());
             if (storeExtensionService.isNeedSendMsgType(logStoreDO.getLogType())) {
                 CompletableFuture.runAsync(() -> sendMessageOnDelete(milogLogtailDo, logStoreDO));
             }
-            CompletableFuture.runAsync(() -> sendMessageOnDelete(milogLogtailDo, logStoreDO));
             tailExtensionService.logTailDelPostProcess(logStoreDO, milogLogtailDo);
-            MilogLogStoreDO storeDO = logStoreDao.queryById(milogLogtailDo.getStoreId());
-            logTailService.deleteConfigRemote(storeDO.getSpaceId(), id, storeDO.getMachineRoom(), LogStructureEnum.TAIL);
-
             return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage());
         } else {
             log.warn("[LogTailService.deleteMilogLogtail] delete MilogLogtail err,id:{}", id);
             return new Result<>(CommonError.UnknownError.getCode(), CommonError.UnknownError.getMessage());
         }
     }
 
+    private void deleteRemoteConfig(Long id, MilogLogTailDo milogLogtailDo) {
+        MilogLogStoreDO storeDO = logStoreDao.queryById(milogLogtailDo.getStoreId());
+        logTailService.deleteConfigRemote(storeDO.getSpaceId(), id, storeDO.getMachineRoom(), LogStructureEnum.TAIL);
+    }
+
     @Override
     public void sendMessageOnDelete(MilogLogTailDo mt, MilogLogStoreDO logStoreDO) {
         log.info("发送删除配置信息：mt:{}", GSON.toJson(mt));

diff --git a/...n-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java b/...n-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java
@@ -33,6 +33,7 @@ public class EsService {
     public EsClient getEsClient() {
         return this.esClient;
     }
+
     public EsService(String esAddr, String token, String catalog, String database) {
         this.esClient = new EsClient(esAddr, token, catalog, database);
     }
@@ -141,7 +142,6 @@ public List<IndexTemplateMetadata> getIndexTemplates(GetIndexTemplatesRequest re
     }
 
 
-
     /**
      * 数据直方图
      *
@@ -155,6 +155,10 @@ public EsClient.EsRet dateHistogram(String indexName, String interval, long star
         return esClient.dateHistogram(indexName, interval, startTime, endTime, builder);
     }
 
+    public EsClient.EsRet dateHistogram(String indexName, String field, String interval, long startTime, long endTime, BoolQueryBuilder builder) throws IOException {
+        return esClient.dateHistogram(indexName, field, interval, startTime, endTime, builder);
+    }
+
     /**
      * 统计
      *