From 6030317088337ac4ea6062eedf1d8fa527b75c1c Mon Sep 17 00:00:00 2001 From: wtt <1136220284@qq.com> Date: Thu, 20 Jul 2023 11:31:19 +0800 Subject: [PATCH 1/3] feat: query es add extension --- .../com/xiaomi/mone/log/parse/CustomLogParser.java | 9 +++++++-- .../xiaomi/mone/log/common/CustomLogParserTest.java | 2 +- .../extension/common/CommonExtensionService.java | 2 ++ .../common/DefaultCommonExtensionService.java | 5 +++++ .../log/manager/service/impl/EsDataServiceImpl.java | 4 ++-- .../manager/service/impl/LogQueryServiceImpl.java | 10 +++++++++- .../impl/MilogMiddlewareConfigServiceImpl.java | 12 ++++++------ .../src/main/resources/config/open.properties | 8 ++++---- .../manager/service/impl/LogTailServiceImplTest.java | 12 ++++++++++++ 9 files changed, 48 insertions(+), 16 deletions(-) diff --git a/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java b/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java index 6db43efec..dab179ca5 100644 --- a/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java +++ b/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java @@ -122,12 +122,17 @@ public List parseLogData(String logData) throws Exception { if (StringUtils.isNotEmpty(list.get(0)) && StringUtils.isNotEmpty(list.get(1))) { parsedData = StringUtils.substringBetween(logData, list.get(0), list.get(1)); } else { - parsedData = logData; + //下一个索引的前部分不为空的话依赖它 + if (i + 1 < mapPattern.size() && StringUtils.isNotEmpty(mapPattern.get(i + 1).get(0))) { + parsedData = StringUtils.substringBetween(logData, "", mapPattern.get(i + 1).get(0)); + } else { + parsedData = logData; + } } if (null == parsedData) { break; } - parsedLogs.add(parsedData); + parsedLogs.add(parsedData.trim()); logData = StrUtil.removePrefix(logData.trim(), String.format("%s%s%s", list.get(0), parsedData, list.get(1)).trim()); } return parsedLogs; diff --git a/hera-all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java b/hera-all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java index e326e4d29..04fc732dd 100644 --- a/hera-all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java +++ b/hera-all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java @@ -43,7 +43,7 @@ public class CustomLogParserTest { @Test public void test1() { Stopwatch stopwatch = Stopwatch.createStarted(); - String keyList = "timestamp:date,level:keyword,traceId:keyword,threadName:text,className:text,line:keyword,methodName:keyword,message:text,podName:keyword,failed:text,taskId:text,apiType:text,uri:text,method:text,rt:text,code:text,sceneId:text,serialId:text,reportId:text,apiId:text,params:text,result:text,errorInfo :text,reqHeaders:text,respHeaders:text,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long"; + String keyList = "timestamp:date,podName:keyword,level:keyword,threadName:text,className:text,line:keyword,methodName:keyword,traceId:keyword,message:text,ip:ip,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long"; String valueList = "0,-1,16,-1,-1,-1,-1,-1,-1,1,2,3,4,5,6,7,8,9,10,11,13,12,17,14,15"; String parseScript = "|"; String logData = ""; diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java index 739e4b950..aadc7ba79 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java @@ -43,4 +43,6 @@ public interface CommonExtensionService { TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal); String queryDateHistogramField(Long storeId); + + String getSearchIndex(Long logStoreId, String esIndex); } diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java index b519007c9..ade12b8ae 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java @@ -75,4 +75,9 @@ public TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal) { public String queryDateHistogramField(Long storeId) { return "timestamp"; } + + @Override + public String getSearchIndex(Long logStoreId, String esIndexName) { + return esIndexName; + } } diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java index 972b74a11..8067d695a 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java @@ -150,7 +150,7 @@ public Result logQuery(LogQuery logQuery) { return Result.failParam("找不到[" + logQuery.getLogstore() + "]对应的数据"); } EsService esService = esCluster.getEsService(milogLogstoreDO.getEsClusterId()); - String esIndexName = milogLogstoreDO.getEsIndex(); + String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), milogLogstoreDO.getEsIndex()); if (esService == null || StringUtils.isEmpty(esIndexName)) { log.warn("[EsDataService.logQuery] logStore:[{}]配置异常", logQuery.getLogstore()); return Result.failParam("logStore配置异常"); @@ -260,7 +260,7 @@ public Result EsStatistic(LogQuery logQuery) throws Exception // get interval String interval = searchLog.esHistogramInterval(logQuery.getEndTime() - logQuery.getStartTime()); EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndex = logStore.getEsIndex(); + String esIndex = commonExtensionService.getSearchIndex(logStore.getId(), logStore.getEsIndex()); if (esService == null || StringUtils.isEmpty(esIndex)) { return Result.failParam("logStore或tail配置异常"); } diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java index 5c147a910..24d6e942c 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java @@ -38,6 +38,8 @@ import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery; import com.xiaomi.mone.log.manager.service.EsDataBaseService; import com.xiaomi.mone.log.manager.service.LogQueryService; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService; +import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory; import com.xiaomi.mone.log.parse.LogParser; import com.xiaomi.youpin.docean.anno.Service; import com.xiaomi.youpin.docean.common.StringUtils; @@ -87,6 +89,12 @@ public class LogQueryServiceImpl implements LogQueryService, LogDataService, EsD private Set hidenFiledSet = new HashSet<>(); + private CommonExtensionService commonExtensionService; + + public void init() { + commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService(); + } + { noHighLightSet.add("logstore"); noHighLightSet.add("logsource"); @@ -122,7 +130,7 @@ public Result logQuery(LogQuery logQuery) { return Result.failParam("找不到[" + logQuery.getLogstore() + "]对应的数据"); } EsService esService = esCluster.getEsService(logStore.getEsClusterId()); - String esIndexName = logStore.getEsIndex(); + String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), logStore.getEsIndex()); if (esService == null || StringUtils.isEmpty(esIndexName)) { log.warn("[EsDataService.logQuery] logstroe:[{}]配置异常", logQuery.getLogstore()); return Result.failParam("logstroe配置异常"); diff --git a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java index e9b2cc9e0..c475b2ec9 100644 --- a/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java +++ b/hera-all/hera-log/log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java @@ -373,11 +373,11 @@ public ResourceUserSimple userResourceList(String regionCode, Integer logTypeCod if (initializedStatus && showStatus) { - List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCode(MiddlewareEnum.ROCKETMQ.getCode(), regionCode); - queryCurrentUserResourceList(resourceUserSimple, milogMiddlewareConfigs, MiddlewareEnum.ROCKETMQ); + List milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCode(resourceExtensionService.getResourceCode(), regionCode); + queryCurrentUserResourceList(resourceUserSimple, milogMiddlewareConfigs, resourceExtensionService.getResourceCode()); List middlewareConfigEss = getESConfigs(regionCode); - queryCurrentUserResourceList(resourceUserSimple, middlewareConfigEss, MiddlewareEnum.ELASTICSEARCH); + queryCurrentUserResourceList(resourceUserSimple, middlewareConfigEss, MiddlewareEnum.ELASTICSEARCH.getCode()); } return resourceUserSimple; } @@ -441,18 +441,18 @@ private List queryMilogEsIndex(List clusterIds, Integer lo } /** - * 查询当前用户的三级部门下的资源列表 + * 查询当前用户的资源列表 * * @param configResource */ private void queryCurrentUserResourceList(ResourceUserSimple configResource, - List milogMiddlewareConfigs, MiddlewareEnum middlewareEnum) { + List milogMiddlewareConfigs, Integer code) { milogMiddlewareConfigs = resourceExtensionService.currentUserConfigFilter(milogMiddlewareConfigs); List valueKeyObjs = milogMiddlewareConfigs.stream() .map(milogMiddlewareConfig -> new ValueKeyObj( milogMiddlewareConfig.getId(), milogMiddlewareConfig.getAlias())) .collect(Collectors.toList()); - if (MiddlewareEnum.ELASTICSEARCH == middlewareEnum) { + if (Objects.equals(MiddlewareEnum.ELASTICSEARCH.getCode(), code)) { configResource.setEsResourceList(valueKeyObjs); return; } diff --git a/hera-all/hera-log/log-manager/src/main/resources/config/open.properties b/hera-all/hera-log/log-manager/src/main/resources/config/open.properties index fc4d807f9..a2ccc7668 100644 --- a/hera-all/hera-log/log-manager/src/main/resources/config/open.properties +++ b/hera-all/hera-log/log-manager/src/main/resources/config/open.properties @@ -1,7 +1,7 @@ -nacosAddr=nacos:80 -defaultNacosAddres=nacos:80 +nacosAddr=nacos.test.b2c.srv:80 +defaultNacosAddres=nacos.test.b2c.srv:80 -nacos_config_dataid=hera_log_manager_open +nacos_config_dataid=hera_log_manager serverName=milog_manager_server_open serverNameHttp=milog_manager_server_open @@ -37,7 +37,7 @@ dubbo_app_name=milog dubbo.group= dubbo.env.group=open dubbo_reg_check=false -dubbo_reg_address=nacos://nacos:80 +dubbo_reg_address=nacos://nacos.test.b2c.srv:80 nacos.config.addrs=nacos:80 #\u53EA\u53D1\u9001mq\uFF0C\u4E0D\u6D88\u8D39\u7684\u65E5\u5FD7\u7C7B\u578B diff --git a/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java b/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java index 76cb33e23..49f5aba4c 100644 --- a/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java +++ b/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java @@ -17,6 +17,8 @@ import com.google.gson.Gson; import com.xiaomi.mone.log.api.enums.LogStructureEnum; +import com.xiaomi.mone.log.common.Result; +import com.xiaomi.mone.log.manager.model.bo.MlogParseParam; import com.xiaomi.youpin.docean.Ioc; import lombok.extern.slf4j.Slf4j; import org.junit.Before; @@ -53,4 +55,14 @@ public void deleteConfigRemoteTest() { logTailService.deleteConfigRemote(spaceId, id, motorRoomEn, logStructureEnum); log.info("deleteConfigRemoteTest"); } + + @Test + public void parseExampleTest() { + MlogParseParam mlogParseParam = new MlogParseParam(); + mlogParseParam.setParseType(5); + mlogParseParam.setParseScript("%s-[%s]-[%s]-%s-[%s]-[%s]-[%s]-%s"); + mlogParseParam.setMsg("2023-07-19 09:48:54.178 [dk-fdf] [10.126.161.114] INFO [http-nio-8051-exec-166] [com.xiaomi.duokan.api.fiction.common.aspect.PerfCountAspectImpl#doAround:57] [f17c2f4e2f1d29d5015931e682abe400] FictionFeignClient.batchQueryFictionScore costTime:4"); + Result objectResult = logTailService.parseExample(mlogParseParam); + log.info("result:{}", objectResult); + } } From 3d742acc7bc645827558343b3665d2c9fc867416 Mon Sep 17 00:00:00 2001 From: wtt <1136220284@qq.com> Date: Thu, 20 Jul 2023 11:32:50 +0800 Subject: [PATCH 2/3] feat: update config --- .../src/main/resources/config/open.properties | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hera-all/hera-log/log-manager/src/main/resources/config/open.properties b/hera-all/hera-log/log-manager/src/main/resources/config/open.properties index a2ccc7668..014864940 100644 --- a/hera-all/hera-log/log-manager/src/main/resources/config/open.properties +++ b/hera-all/hera-log/log-manager/src/main/resources/config/open.properties @@ -1,7 +1,7 @@ -nacosAddr=nacos.test.b2c.srv:80 -defaultNacosAddres=nacos.test.b2c.srv:80 +nacosAddr=nacos:80 +defaultNacosAddres=nacos:80 -nacos_config_dataid=hera_log_manager +nacos_config_dataid=hera_log_manager_open serverName=milog_manager_server_open serverNameHttp=milog_manager_server_open @@ -37,7 +37,7 @@ dubbo_app_name=milog dubbo.group= dubbo.env.group=open dubbo_reg_check=false -dubbo_reg_address=nacos://nacos.test.b2c.srv:80 +dubbo_reg_address=nacos://nacos:80 nacos.config.addrs=nacos:80 #\u53EA\u53D1\u9001mq\uFF0C\u4E0D\u6D88\u8D39\u7684\u65E5\u5FD7\u7C7B\u578B @@ -56,4 +56,3 @@ tpc_node_code=logger filter_urls=queryAgentK8sIp,queryAgentConfig,matrixLogQuery agent.heart.senders=zhangsan tpc.devMode=false - From 8258fc56c8f5179d2cd49e48ed59bdcf3578437c Mon Sep 17 00:00:00 2001 From: wtt <1136220284@qq.com> Date: Thu, 20 Jul 2023 11:40:27 +0800 Subject: [PATCH 3/3] feat: update --- .../mone/log/manager/service/impl/LogTailServiceImplTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java b/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java index 49f5aba4c..7442072f0 100644 --- a/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java +++ b/hera-all/hera-log/log-manager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java @@ -61,7 +61,7 @@ public void parseExampleTest() { MlogParseParam mlogParseParam = new MlogParseParam(); mlogParseParam.setParseType(5); mlogParseParam.setParseScript("%s-[%s]-[%s]-%s-[%s]-[%s]-[%s]-%s"); - mlogParseParam.setMsg("2023-07-19 09:48:54.178 [dk-fdf] [10.126.161.114] INFO [http-nio-8051-exec-166] [com.xiaomi.duokan.api.fiction.common.aspect.PerfCountAspectImpl#doAround:57] [f17c2f4e2f1d29d5015931e682abe400] FictionFeignClient.batchQueryFictionScore costTime:4"); + mlogParseParam.setMsg("2023-07-19 09:48:54.178 [dk-fdf] [127.0.0.1] INFO [http-nio-8051-exec-166] [com.xiaomi.duokan.api.fiction.common.aspect.PerfCountAspectImpl#doAround:57] [f17c2f4e2f1d29d5015931e682abe400] FictionFeignClient.batchQueryFictionScore costTime:4"); Result objectResult = logTailService.parseExample(mlogParseParam); log.info("result:{}", objectResult); }