From 9f66fc7ca61fa9057139035da6c8033a9666aaea Mon Sep 17 00:00:00 2001 From: wtt <1136220284@qq.com> Date: Thu, 13 Jul 2023 11:21:14 +0800 Subject: [PATCH 1/2] feat: delete tail process --- hera-all/hera-log/log-manager/pom.xml | 25 ++++++++++++++++++ .../mone/log/manager/domain/AnalyseLog.java | 2 +- .../mone/log/manager/domain/SearchLog.java | 26 +++++++++---------- .../common/CommonExtensionService.java | 10 +++++++ .../common/DefaultCommonExtensionService.java | 22 ++++++++++++++++ .../service/impl/EsDataServiceImpl.java | 23 +++++++++------- .../service/impl/LogQueryServiceImpl.java | 2 +- .../service/impl/LogTailServiceImpl.java | 12 ++++++--- 8 files changed, 93 insertions(+), 29 deletions(-) diff --git a/hera-all/hera-log/log-manager/pom.xml b/hera-all/hera-log/log-manager/pom.xml index a87354ef4..4f0a0a563 100644 --- a/hera-all/hera-log/log-manager/pom.xml +++ b/hera-all/hera-log/log-manager/pom.xml @@ -36,10 +36,22 @@ 5.3.25 + + run.mone + spring-context-support + 1.0.10-mone-SNAPSHOT + + run.mone dubbo 2.7.12-mone-v8-SNAPSHOT + + + com.alibaba.spring + spring-context-support + + @@ -102,10 +114,23 @@ + + run.mone + es + 1.5-SNAPSHOT + compile + + run.mone docean-plugin-es 1.4-SNAPSHOT + + + run.mone + es + + diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java index 851c478d0..18348ad48 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/AnalyseLog.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.aggregations.AggregationBuilder; diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java index b22557c45..90ae23ca6 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/domain/SearchLog.java @@ -17,6 +17,8 @@ import com.xiaomi.mone.log.manager.model.vo.LogContextQuery; import com.xiaomi.mone.log.manager.model.vo.LogQuery; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; import com.xiaomi.mone.log.manager.service.statement.StatementMatchParseFactory; import com.xiaomi.youpin.docean.anno.Service; import com.xiaomi.youpin.docean.common.DoceanConfig; @@ -38,6 +40,13 @@ @Slf4j @Service public class SearchLog { + + private CommonExtensionService commonExtensionService; + + public void init() { + commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); + } + /** * 获取查询参数 * @@ -58,16 +67,14 @@ public BoolQueryBuilder getQueryBuilder(LogQuery logQuery, List keyList) return boolQueryBuilder; } - private static BoolQueryBuilder buildCommonBuilder(LogQuery logQuery) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(logQuery.getStartTime()).to(logQuery.getEndTime())); - boolQueryBuilder.filter(QueryBuilders.termQuery("logstore", logQuery.getLogstore())); + private BoolQueryBuilder buildCommonBuilder(LogQuery logQuery) { + BoolQueryBuilder boolQueryBuilder = commonExtensionService.commonRangeQuery(logQuery); // 支持tail多选 if (StringUtils.isNotEmpty(logQuery.getTail())) { BoolQueryBuilder tailQueryBuilder = QueryBuilders.boolQuery(); String[] tailLimitArray = logQuery.getTail().split(","); for (String tail : tailLimitArray) { - tailQueryBuilder.should(QueryBuilders.termQuery("tail", tail)); + tailQueryBuilder.should(commonExtensionService.multipleChooseBuilder(logQuery.getStoreId(), tail)); } tailQueryBuilder.minimumShouldMatch(1); boolQueryBuilder.filter(tailQueryBuilder); @@ -283,14 +290,7 @@ private static int getEndIndex(String querytext) { } public boolean isLegalParam(LogContextQuery param) { - if (param == null - || StringUtils.isEmpty(param.getLogstore()) - || StringUtils.isEmpty(param.getIp()) - || StringUtils.isEmpty(param.getFileName()) - || param.getLineNumber() == null - || StringUtils.isEmpty(param.getTimestamp()) - || param.getType() == null - || param.getPageSize() == null) { + if (param == null || StringUtils.isEmpty(param.getLogstore()) || StringUtils.isEmpty(param.getIp()) || StringUtils.isEmpty(param.getFileName()) || param.getLineNumber() == null || StringUtils.isEmpty(param.getTimestamp()) || param.getType() == null || param.getPageSize() == null) { return false; } return true; diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java index b46fca326..739e4b950 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java @@ -15,6 +15,10 @@ */ package com.xiaomi.mone.log.manager.service.extension.common; +import com.xiaomi.mone.log.manager.model.vo.LogQuery; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; + /** * @author wtt * @version 1.0 @@ -33,4 +37,10 @@ public interface CommonExtensionService { String getMachineRoomName(String machineRoomEn); boolean middlewareEnumValid(Integer type); + + BoolQueryBuilder commonRangeQuery(LogQuery logQuery); + + TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal); + + String queryDateHistogramField(Long storeId); } diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java index 57bc6b5e2..b519007c9 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java @@ -17,8 +17,12 @@ import com.xiaomi.mone.log.api.enums.MachineRegionEnum; import com.xiaomi.mone.log.api.enums.MiddlewareEnum; +import com.xiaomi.mone.log.manager.model.vo.LogQuery; import com.xiaomi.youpin.docean.anno.Service; import lombok.extern.slf4j.Slf4j; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.TermQueryBuilder; import static com.xiaomi.mone.log.common.Constant.DEFAULT_STREAM_SERVER_NAME; import static com.xiaomi.mone.log.common.Constant.LOG_MANAGE_PREFIX; @@ -53,4 +57,22 @@ public String getMachineRoomName(String machineRoomEn) { public boolean middlewareEnumValid(Integer type) { return MiddlewareEnum.ROCKETMQ.getCode().equals(type); } + + @Override + public BoolQueryBuilder commonRangeQuery(LogQuery logQuery) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(QueryBuilders.rangeQuery("timestamp").from(logQuery.getStartTime()).to(logQuery.getEndTime())); + boolQueryBuilder.filter(QueryBuilders.termQuery("logstore", logQuery.getLogstore())); + return boolQueryBuilder; + } + + @Override + public TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal) { + return QueryBuilders.termQuery("tail", chooseVal); + } + + @Override + public String queryDateHistogramField(Long storeId) { + return "timestamp"; + } } diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java index faaa66c2a..972b74a11 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java @@ -21,7 +21,6 @@ import com.xiaomi.mone.log.api.model.dto.TraceLogDTO; import com.xiaomi.mone.log.api.model.vo.TraceLogQuery; import com.xiaomi.mone.log.api.service.LogDataService; -import com.xiaomi.mone.log.common.HeraLocalCache; import com.xiaomi.mone.log.common.Result; import com.xiaomi.mone.log.exception.CommonError; import com.xiaomi.mone.log.manager.common.context.MoneUserContext; @@ -44,6 +43,8 @@ import com.xiaomi.mone.log.manager.model.vo.TraceAppLogUrlQuery; import com.xiaomi.mone.log.manager.service.EsDataBaseService; import com.xiaomi.mone.log.manager.service.EsDataService; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; import com.xiaomi.mone.log.parse.LogParser; import com.xiaomi.youpin.docean.anno.Service; import com.xiaomi.youpin.docean.common.StringUtils; @@ -55,8 +56,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.core.CountRequest; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; @@ -106,6 +106,12 @@ public class EsDataServiceImpl implements EsDataService, LogDataService, EsDataB @Reference(interfaceClass = LogDataService.class, group = "$dubbo.youpin.group", check = false, timeout = 5000) private LogDataService logDataService; + private CommonExtensionService commonExtensionService; + + public void init() { + commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); + } + private Set noHighLightSet = new HashSet<>(); private Set hidenFiledSet = new HashSet<>(); @@ -247,7 +253,7 @@ public Result EsStatistic(LogQuery logQuery) throws Exception try { EsStatisticResult result = new EsStatisticResult(); result.setName(constractEsStatisticRet(logQuery)); - MilogLogStoreDO logStore = logstoreDao.getByName(logQuery.getLogstore()); + MilogLogStoreDO logStore = logstoreDao.queryById(logQuery.getStoreId()); if (logStore == null) { return new Result<>(CommonError.UnknownError.getCode(), "not found logstore", null); } @@ -260,7 +266,8 @@ public Result EsStatistic(LogQuery logQuery) throws Exception } if (!StringUtils.isEmpty(interval)) { BoolQueryBuilder queryBuilder = searchLog.getQueryBuilder(logQuery, getKeyColonPrefix(logStore.getKeyList())); - EsClient.EsRet esRet = esService.dateHistogram(esIndex, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder); + String histogramField = commonExtensionService.queryDateHistogramField(logQuery.getStoreId()); + EsClient.EsRet esRet = esService.dateHistogram(esIndex, histogramField, interval, logQuery.getStartTime(), logQuery.getEndTime(), queryBuilder); result.setCounts(esRet.getCounts()); result.setTimestamps(esRet.getTimestamps()); result.setQueryBuilder(queryBuilder); @@ -448,11 +455,7 @@ public void logExport(LogQuery logQuery) throws Exception { int maxLogNum = 10000; logQuery.setPageSize(maxLogNum); Result logDTOResult = this.logQuery(logQuery); - List> exportData = - logDTOResult.getCode() != CommonError.Success.getCode() - || logDTOResult.getData().getLogDataDTOList() == null - || logDTOResult.getData().getLogDataDTOList().isEmpty() ? - null : logDTOResult.getData().getLogDataDTOList().stream().map(logDataDto -> ExportUtils.SplitTooLongContent(logDataDto)).collect(Collectors.toList()); + List> exportData = logDTOResult.getCode() != CommonError.Success.getCode() || logDTOResult.getData().getLogDataDTOList() == null || logDTOResult.getData().getLogDataDTOList().isEmpty() ? null : logDTOResult.getData().getLogDataDTOList().stream().map(logDataDto -> ExportUtils.SplitTooLongContent(logDataDto)).collect(Collectors.toList()); HSSFWorkbook excel = ExportExcel.HSSFWorkbook4Map(exportData, generateTitle(logQuery)); // 下载 String fileName = String.format("%s_log.xls", logQuery.getLogstore()); diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java index c75bddbc2..5c147a910 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java @@ -47,7 +47,7 @@ import org.elasticsearch.ElasticsearchStatusException; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.core.TimeValue; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java index 3a5b66104..894e55ba2 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImpl.java @@ -469,16 +469,15 @@ public Result deleteLogTail(Long id) { if (StringUtils.isNotEmpty(validMsg)) { return new Result<>(CommonError.ParamsError.getCode(), validMsg); } + + deleteRemoteConfig(id, milogLogtailDo); + if (milogLogtailDao.deleteMilogLogtail(id)) { MilogLogStoreDO logStoreDO = logStoreDao.queryById(milogLogtailDo.getStoreId()); if (storeExtensionService.isNeedSendMsgType(logStoreDO.getLogType())) { CompletableFuture.runAsync(() -> sendMessageOnDelete(milogLogtailDo, logStoreDO)); } - CompletableFuture.runAsync(() -> sendMessageOnDelete(milogLogtailDo, logStoreDO)); tailExtensionService.logTailDelPostProcess(logStoreDO, milogLogtailDo); - MilogLogStoreDO storeDO = logStoreDao.queryById(milogLogtailDo.getStoreId()); - logTailService.deleteConfigRemote(storeDO.getSpaceId(), id, storeDO.getMachineRoom(), LogStructureEnum.TAIL); - return new Result<>(CommonError.Success.getCode(), CommonError.Success.getMessage()); } else { log.warn("[LogTailService.deleteMilogLogtail] delete MilogLogtail err,id:{}", id); @@ -486,6 +485,11 @@ public Result deleteLogTail(Long id) { } } + private void deleteRemoteConfig(Long id, MilogLogTailDo milogLogtailDo) { + MilogLogStoreDO storeDO = logStoreDao.queryById(milogLogtailDo.getStoreId()); + logTailService.deleteConfigRemote(storeDO.getSpaceId(), id, storeDO.getMachineRoom(), LogStructureEnum.TAIL); + } + @Override public void sendMessageOnDelete(MilogLogTailDo mt, MilogLogStoreDO logStoreDO) { log.info("发送删除配置信息：mt:{}", GSON.toJson(mt)); From 2ac83dd6acaa89b3c691f42c373000b9b9dd6af9 Mon Sep 17 00:00:00 2001 From: wtt <1136220284@qq.com> Date: Fri, 14 Jul 2023 14:31:00 +0800 Subject: [PATCH 2/2] feat:update --- .../java/com/xiaomi/youpin/docean/plugin/es/EsService.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/jcommon/docean-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java b/jcommon/docean-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java index 675169bb9..656c14fc4 100644 --- a/jcommon/docean-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java +++ b/jcommon/docean-plugin/docean-plugin-es/src/main/java/com/xiaomi/youpin/docean/plugin/es/EsService.java @@ -33,6 +33,7 @@ public class EsService { public EsClient getEsClient() { return this.esClient; } + public EsService(String esAddr, String token, String catalog, String database) { this.esClient = new EsClient(esAddr, token, catalog, database); } @@ -141,7 +142,6 @@ public List getIndexTemplates(GetIndexTemplatesRequest re } - /** * 数据直方图 * @@ -155,6 +155,10 @@ public EsClient.EsRet dateHistogram(String indexName, String interval, long star return esClient.dateHistogram(indexName, interval, startTime, endTime, builder); } + public EsClient.EsRet dateHistogram(String indexName, String field, String interval, long startTime, long endTime, BoolQueryBuilder builder) throws IOException { + return esClient.dateHistogram(indexName, field, interval, startTime, endTime, builder); + } + /** * 统计 *