XiaoMi · gaoxh · Jul 20, 2023 · Jul 20, 2023 · Jul 20, 2023 · Jul 20, 2023
diff --git a/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java b/hera-all/hera-log/log-common/src/main/java/com/xiaomi/mone/log/parse/CustomLogParser.java
@@ -122,12 +122,17 @@ public List<String> parseLogData(String logData) throws Exception {
             if (StringUtils.isNotEmpty(list.get(0)) && StringUtils.isNotEmpty(list.get(1))) {
                 parsedData = StringUtils.substringBetween(logData, list.get(0), list.get(1));
             } else {
-                parsedData = logData;
+                //下一个索引的前部分不为空的话依赖它
+                if (i + 1 < mapPattern.size() && StringUtils.isNotEmpty(mapPattern.get(i + 1).get(0))) {
+                    parsedData = StringUtils.substringBetween(logData, "", mapPattern.get(i + 1).get(0));
+                } else {
+                    parsedData = logData;
+                }
             }
             if (null == parsedData) {
                 break;
             }
-            parsedLogs.add(parsedData);
+            parsedLogs.add(parsedData.trim());
             logData = StrUtil.removePrefix(logData.trim(), String.format("%s%s%s", list.get(0), parsedData, list.get(1)).trim());
         }
         return parsedLogs;

diff --git a/...all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java b/...all/hera-log/log-common/src/test/java/com/xiaomi/mone/log/common/CustomLogParserTest.java
@@ -43,7 +43,7 @@ public class CustomLogParserTest {
     @Test
     public void test1() {
         Stopwatch stopwatch = Stopwatch.createStarted();
-        String keyList = "timestamp:date,level:keyword,traceId:keyword,threadName:text,className:text,line:keyword,methodName:keyword,message:text,podName:keyword,failed:text,taskId:text,apiType:text,uri:text,method:text,rt:text,code:text,sceneId:text,serialId:text,reportId:text,apiId:text,params:text,result:text,errorInfo :text,reqHeaders:text,respHeaders:text,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long";
+        String keyList = "timestamp:date,podName:keyword,level:keyword,threadName:text,className:text,line:keyword,methodName:keyword,traceId:keyword,message:text,ip:ip,logstore:keyword,logsource:keyword,mqtopic:keyword,mqtag:keyword,logip:keyword,tail:keyword,linenumber:long";
         String valueList = "0,-1,16,-1,-1,-1,-1,-1,-1,1,2,3,4,5,6,7,8,9,10,11,13,12,17,14,15";
         String parseScript = "|";
         String logData = "";

diff --git a/...ain/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java b/...ain/java/com/xiaomi/mone/log/manager/service/extension/common/CommonExtensionService.java
@@ -43,4 +43,6 @@ public interface CommonExtensionService {
     TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal);
 
     String queryDateHistogramField(Long storeId);
+
+    String getSearchIndex(Long logStoreId, String esIndex);
 }
diff --git a/...a/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java b/...a/com/xiaomi/mone/log/manager/service/extension/common/DefaultCommonExtensionService.java
@@ -75,4 +75,9 @@ public TermQueryBuilder multipleChooseBuilder(Long storeId, String chooseVal) {
     public String queryDateHistogramField(Long storeId) {
         return "timestamp";
     }
+
+    @Override
+    public String getSearchIndex(Long logStoreId, String esIndexName) {
+        return esIndexName;
+    }
 }
diff --git a/...log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java b/...log-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/EsDataServiceImpl.java
@@ -150,7 +150,7 @@ public Result<LogDTO> logQuery(LogQuery logQuery) {
                 return Result.failParam("找不到[" + logQuery.getLogstore() + "]对应的数据");
             }
             EsService esService = esCluster.getEsService(milogLogstoreDO.getEsClusterId());
-            String esIndexName = milogLogstoreDO.getEsIndex();
+            String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), milogLogstoreDO.getEsIndex());
             if (esService == null || StringUtils.isEmpty(esIndexName)) {
                 log.warn("[EsDataService.logQuery] logStore:[{}]配置异常", logQuery.getLogstore());
                 return Result.failParam("logStore配置异常");
@@ -260,7 +260,7 @@ public Result<EsStatisticResult> EsStatistic(LogQuery logQuery) throws Exception
             // get interval
             String interval = searchLog.esHistogramInterval(logQuery.getEndTime() - logQuery.getStartTime());
             EsService esService = esCluster.getEsService(logStore.getEsClusterId());
-            String esIndex = logStore.getEsIndex();
+            String esIndex = commonExtensionService.getSearchIndex(logStore.getId(), logStore.getEsIndex());
             if (esService == null || StringUtils.isEmpty(esIndex)) {
                 return Result.failParam("logStore或tail配置异常");
             }

diff --git a/...g-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java b/...g-manager/src/main/java/com/xiaomi/mone/log/manager/service/impl/LogQueryServiceImpl.java
@@ -38,6 +38,8 @@
 import com.xiaomi.mone.log.manager.model.vo.RegionTraceLogQuery;
 import com.xiaomi.mone.log.manager.service.EsDataBaseService;
 import com.xiaomi.mone.log.manager.service.LogQueryService;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionService;
+import com.xiaomi.mone.log.manager.service.extension.common.CommonExtensionServiceFactory;
 import com.xiaomi.mone.log.parse.LogParser;
 import com.xiaomi.youpin.docean.anno.Service;
 import com.xiaomi.youpin.docean.common.StringUtils;
@@ -87,6 +89,12 @@ public class LogQueryServiceImpl implements LogQueryService, LogDataService, EsD
 
     private Set<String> hidenFiledSet = new HashSet<>();
 
+    private CommonExtensionService commonExtensionService;
+
+    public void init() {
+        commonExtensionService = CommonExtensionServiceFactory.getCommonExtensionService();
+    }
+
     {
         noHighLightSet.add("logstore");
         noHighLightSet.add("logsource");
@@ -122,7 +130,7 @@ public Result<LogDTO> logQuery(LogQuery logQuery) {
                 return Result.failParam("找不到[" + logQuery.getLogstore() + "]对应的数据");
             }
             EsService esService = esCluster.getEsService(logStore.getEsClusterId());
-            String esIndexName = logStore.getEsIndex();
+            String esIndexName = commonExtensionService.getSearchIndex(logQuery.getStoreId(), logStore.getEsIndex());
             if (esService == null || StringUtils.isEmpty(esIndexName)) {
                 log.warn("[EsDataService.logQuery] logstroe:[{}]配置异常", logQuery.getLogstore());
                 return Result.failParam("logstroe配置异常");

diff --git a/.../main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java b/.../main/java/com/xiaomi/mone/log/manager/service/impl/MilogMiddlewareConfigServiceImpl.java
@@ -373,11 +373,11 @@ public ResourceUserSimple userResourceList(String regionCode, Integer logTypeCod
 
         if (initializedStatus && showStatus) {
 
-            List<MilogMiddlewareConfig> milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCode(MiddlewareEnum.ROCKETMQ.getCode(), regionCode);
-            queryCurrentUserResourceList(resourceUserSimple, milogMiddlewareConfigs, MiddlewareEnum.ROCKETMQ);
+            List<MilogMiddlewareConfig> milogMiddlewareConfigs = milogMiddlewareConfigDao.queryByResourceCode(resourceExtensionService.getResourceCode(), regionCode);
+            queryCurrentUserResourceList(resourceUserSimple, milogMiddlewareConfigs, resourceExtensionService.getResourceCode());
 
             List<MilogMiddlewareConfig> middlewareConfigEss = getESConfigs(regionCode);
-            queryCurrentUserResourceList(resourceUserSimple, middlewareConfigEss, MiddlewareEnum.ELASTICSEARCH);
+            queryCurrentUserResourceList(resourceUserSimple, middlewareConfigEss, MiddlewareEnum.ELASTICSEARCH.getCode());
         }
         return resourceUserSimple;
     }
@@ -441,18 +441,18 @@ private List<MilogEsIndexDO> queryMilogEsIndex(List<Long> clusterIds, Integer lo
     }
 
     /**
-     * 查询当前用户的三级部门下的资源列表
+     * 查询当前用户的资源列表
      *
      * @param configResource
      */
     private void queryCurrentUserResourceList(ResourceUserSimple configResource,
-                                              List<MilogMiddlewareConfig> milogMiddlewareConfigs, MiddlewareEnum middlewareEnum) {
+                                              List<MilogMiddlewareConfig> milogMiddlewareConfigs, Integer code) {
         milogMiddlewareConfigs = resourceExtensionService.currentUserConfigFilter(milogMiddlewareConfigs);
         List valueKeyObjs = milogMiddlewareConfigs.stream()
                 .map(milogMiddlewareConfig -> new ValueKeyObj(
                         milogMiddlewareConfig.getId(), milogMiddlewareConfig.getAlias()))
                 .collect(Collectors.toList());
-        if (MiddlewareEnum.ELASTICSEARCH == middlewareEnum) {
+        if (Objects.equals(MiddlewareEnum.ELASTICSEARCH.getCode(), code)) {
             configResource.setEsResourceList(valueKeyObjs);
             return;
         }

diff --git a/hera-all/hera-log/log-manager/src/main/resources/config/open.properties b/hera-all/hera-log/log-manager/src/main/resources/config/open.properties
@@ -56,4 +56,3 @@ tpc_node_code=logger
 filter_urls=queryAgentK8sIp,queryAgentConfig,matrixLogQuery
 agent.heart.senders=zhangsan
 tpc.devMode=false
-
diff --git a/...anager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java b/...anager/src/test/java/com/xiaomi/mone/log/manager/service/impl/LogTailServiceImplTest.java
@@ -17,6 +17,8 @@
 
 import com.google.gson.Gson;
 import com.xiaomi.mone.log.api.enums.LogStructureEnum;
+import com.xiaomi.mone.log.common.Result;
+import com.xiaomi.mone.log.manager.model.bo.MlogParseParam;
 import com.xiaomi.youpin.docean.Ioc;
 import lombok.extern.slf4j.Slf4j;
 import org.junit.Before;
@@ -53,4 +55,14 @@ public void deleteConfigRemoteTest() {
         logTailService.deleteConfigRemote(spaceId, id, motorRoomEn, logStructureEnum);
         log.info("deleteConfigRemoteTest");
     }
+
+    @Test
+    public void parseExampleTest() {
+        MlogParseParam mlogParseParam = new MlogParseParam();
+        mlogParseParam.setParseType(5);
+        mlogParseParam.setParseScript("%s-[%s]-[%s]-%s-[%s]-[%s]-[%s]-%s");
+        mlogParseParam.setMsg("2023-07-19 09:48:54.178 [dk-fdf] [127.0.0.1] INFO [http-nio-8051-exec-166] [com.xiaomi.duokan.api.fiction.common.aspect.PerfCountAspectImpl#doAround:57] [f17c2f4e2f1d29d5015931e682abe400] FictionFeignClient.batchQueryFictionScore costTime:4");
+        Result<Object> objectResult = logTailService.parseExample(mlogParseParam);
+        log.info("result:{}", objectResult);
+    }
 }