Skip to content

Commit

Permalink
basic alarm distinguish docker and k8s
Browse files Browse the repository at this point in the history
  • Loading branch information
@gaoxihui
gaoxihui committed Jul 24, 2023
1 parent 9d651d0 commit 1125c53
Showing 1 changed file with 54 additions and 32 deletions.
86 changes: 54 additions & 32 deletions ...onitor-service/src/main/java/com/xiaomi/mone/monitor/service/prometheus/AlarmService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -625,6 +625,11 @@ public String getContainerLoadAlarmExpr(Integer projectId,String projectName,Str
exprBuilder.append("avg_over_time(container_cpu_load_average_10s");
exprBuilder.append("{system='mione',");
exprBuilder.append("image!='',");
if(isK8s){
exprBuilder.append("name=~'k8s.*',");
}else {
exprBuilder.append("name!~'k8s.*',");
}

String labelProperties = getEnvLabelProperties(ruleData);
if(StringUtils.isNotBlank(labelProperties)){
Expand Down Expand Up @@ -657,6 +662,11 @@ public String getContainerCpuAlarmExpr(Integer projectId,String projectName,Stri

exprBuilder.append("image!='',");
exprBuilder.append("system='mione',");
if(isK8s){
exprBuilder.append("name=~'k8s.*',");
}else{
exprBuilder.append("name!~'k8s.*',");
}

String labelProperties = getEnvLabelProperties(ruleData);
if(StringUtils.isNotBlank(labelProperties)){
Expand All @@ -672,20 +682,26 @@ public String getContainerCpuAlarmExpr(Integer projectId,String projectName,Stri
}
public String getContainerCpuResourceAlarmExpr(Integer projectId,String projectName,String op,double value,boolean isK8s,AlarmRuleData ruleData){

// String jobLabelValue = prometheusAlarmEnv.equals("production") ? "mione-online-china.*|mione-online-youpin.*" : "mione-staging-china.*|mione-staging-youpin.*";

StringBuilder exprBuilder = new StringBuilder();
exprBuilder.append("sum(irate(container_cpu_usage_seconds_total{");
exprBuilder.append("image!='',");
exprBuilder.append("system='mione',");
if(isK8s){
exprBuilder.append("name=~'k8s.*',");
}else{
exprBuilder.append("name!~'k8s.*',");
}

//mimonitor视为全局配置
if(!projectName.equals("mimonitor")){
exprBuilder.append("application='").append(projectId).append("_").append(projectName.replaceAll("-","_")).append("',");
}

String labelProperties = getEnvLabelProperties(ruleData);
if(StringUtils.isNotBlank(labelProperties)){
exprBuilder.append(labelProperties).append(",");
exprBuilder.append(labelProperties);
}

exprBuilder.append("application='").append(projectId).append("_").append(projectName.replaceAll("-","_")).append("'");
// exprBuilder.append("job=~'").append(jobLabelValue).append("',");
exprBuilder.append("}[1d])) without (cpu) * 100");
exprBuilder.append(op).append(value);
log.info("getContainerCpuResourceAlarmExpr param: projectId:{}, projectName:{}, op:{},value:{}, return:{}",projectId, projectName, op,value, exprBuilder.toString());
Expand All @@ -699,6 +715,11 @@ public String getContainerMemAlarmExpr(Integer projectId,String projectName,Stri

exprBuilder.append("image!='',");
exprBuilder.append("system='mione',");
if(isK8s){
exprBuilder.append("name=~'k8s.*',");
}else {
exprBuilder.append("name!~'k8s.*',");
}

String labelProperties = getEnvLabelProperties(ruleData);
if(StringUtils.isNotBlank(labelProperties)){
Expand Down Expand Up @@ -734,37 +755,36 @@ public String getContainerMemReourceAlarmExpr(Integer projectId,String projectNa
exprBuilder.append(labelProperties).append(",");
}

if(projectName.equals("mimonitor")){
if(isK8s){
exprBuilder.append("name =~'k8s.*'");
}else{
exprBuilder.append("container_label_PROJECT_ID!='',name !~'k8s.*'");
}

if(isK8s){
exprBuilder.append("name =~'k8s.*',");
}else{
exprBuilder.append("application='").append(projectId).append("_").append(projectName).append("'");
exprBuilder.append("container_label_PROJECT_ID!='',name !~'k8s.*',");
}

//mimonitor视为全局配置
if(!projectName.equals("mimonitor")){
exprBuilder.append("application='").append(projectId).append("_").append(projectName.replaceAll("-","_")).append("',");
}


// exprBuilder.append("job=~'").append(jobLabelValue).append("',");
exprBuilder.append("}[1d])) by (application,ip,job,name,system,instance,id,serverEnv) / ");
exprBuilder.append("sum(avg_over_time(container_spec_memory_limit_bytes{");
exprBuilder.append("image!='',");
exprBuilder.append("system='mione',");
if(StringUtils.isNotBlank(labelProperties)){
exprBuilder.append(labelProperties).append(",");

if(isK8s){
exprBuilder.append("name =~'k8s.*',");
}else{
exprBuilder.append("container_label_PROJECT_ID!='',name !~'k8s.*',");
}
// exprBuilder.append("job=~'").append(jobLabelValue).append("',");

if(projectName.equals("mimonitor")){
if(isK8s){
exprBuilder.append("name =~'k8s.*'");
}else{
exprBuilder.append("container_label_PROJECT_ID!='',name !~'k8s.*'");
}
//mimonitor视为全局配置
if(!projectName.equals("mimonitor")){
exprBuilder.append("application='").append(projectId).append("_").append(projectName.replaceAll("-","_")).append("',");
}

}else {
exprBuilder.append("application='").append(projectId).append("_").append(projectName).append("'");
if(StringUtils.isNotBlank(labelProperties)){
exprBuilder.append(labelProperties);
}

exprBuilder.append("}[1d])) by (container_label_PROJECT_ID,application,ip,job,name,system,instance,id,serverEnv,serverZone)) * 100");
Expand Down Expand Up @@ -792,9 +812,6 @@ public String getContainerDiskReourceAlarmExpr(Integer projectId,String projectN

public String getK8sCpuAvgUsageAlarmExpr(Integer projectId,String projectName,String op,double value,AlarmRuleData ruleData){


// String jobLabelValue = prometheusAlarmEnv.equals("production") ? "mione-online-china.*|mione-online-youpin.*" : "mione-staging-china.*|mione-staging-youpin.*";

StringBuilder exprBuilder = new StringBuilder();
exprBuilder.append("sum(irate(container_cpu_usage_seconds_total{");
exprBuilder.append("image!='',system='mione',");
Expand All @@ -804,7 +821,8 @@ public String getK8sCpuAvgUsageAlarmExpr(Integer projectId,String projectName,St
exprBuilder.append(labelProperties).append(",");
}

// exprBuilder.append("job=~'").append(jobLabelValue).append("',");
//k8s标识
exprBuilder.append("name=~'").append("k8s.*").append("',");
exprBuilder.append("application='").append(projectId).append("_").append(projectName).append("'");

exprBuilder.append("}[1m])) without (cpu) * 100 ");
Expand All @@ -817,15 +835,14 @@ public String getK8sCpuAvgUsageAlarmExpr(Integer projectId,String projectName,St
exprBuilder.append(labelProperties).append(",");
}

// exprBuilder.append("job=~'").append(jobLabelValue).append("',");
exprBuilder.append("application='").append(projectId).append("_").append(projectName).append("'");
exprBuilder.append("}");
exprBuilder.append("/");

exprBuilder.append("container_spec_cpu_period{");
exprBuilder.append("system='mione',");
// exprBuilder.append("job=~'").append(jobLabelValue).append("',");
// exprBuilder.append("container=~'").append(projectId).append("-0-.*").append("'");
//k8s标识
exprBuilder.append("name=~'").append("k8s.*").append("',");
exprBuilder.append("application='").append(projectId).append("_").append(projectName).append("'");
exprBuilder.append("}");
exprBuilder.append(")");
Expand Down Expand Up @@ -857,6 +874,11 @@ public String getContainerCountAlarmExpr(Integer projectId,String projectName,St
exprBuilder.append("count(sum_over_time(container_spec_memory_limit_bytes{");
exprBuilder.append("image!='',");
exprBuilder.append("system='mione',");
if(isK8s){
exprBuilder.append("name=~'k8s.*',");
}else {
exprBuilder.append("name!~'k8s.*',");
}

String labelProperties = getEnvLabelProperties(ruleData);
if(StringUtils.isNotBlank(labelProperties)){
Expand Down

0 comments on commit 1125c53

Please sign in to comment.