fix: scrapeconfig stale check when fix-time cron is used

Author: adityathebe

pkg/health/health_fixtures_test.go

@@ -16,6 +16,10 @@ func TestFixtures(t *testing.T) {
 	}
 
 	for _, file := range files {
+		// if file != "testdata/Kubernetes/MissionControl/scrapeConfig-minor-delay.yaml" {
+		// 	continue
+		// }
+
 		testFixture(t, file)
 	}
 }

pkg/health/health_flanksource.go

@@ -122,11 +122,11 @@ func getScrapeConfigHealth(obj *unstructured.Unstructured) (*HealthStatus, error
 			return nil, fmt.Errorf("failed to parse lastRun timestamp: %w", err)
 		}
 
-		var schedule time.Duration
+		var nextRuntime time.Time
 		if scheduleRaw, _, err := unstructured.NestedString(obj.Object, "spec", "schedule"); err != nil {
 			return nil, fmt.Errorf("failed to parse scraper schedule: %w", err)
 		} else if scheduleRaw == "" {
-			schedule = time.Hour // The default schedule
+			nextRuntime = parsedLastRuntime.Add(time.Hour) // The default schedule
 		} else {
 			parsedSchedule, err := cron.ParseStandard(scheduleRaw)
 			if err != nil {
@@ -137,18 +137,18 @@ func getScrapeConfigHealth(obj *unstructured.Unstructured) (*HealthStatus, error
 				}, nil
 			}
 
-			schedule = time.Until(parsedSchedule.Next(time.Now()))
+			nextRuntime = parsedSchedule.Next(parsedLastRuntime)
 		}
 
-		elapsed := time.Since(parsedLastRuntime)
-		if elapsed > schedule*2 {
-			status.Health = HealthUnhealthy
+		// If the ScrapeConfig is few minutes behind the schedule, it's not healthy
+		if time.Since(nextRuntime) > time.Minute*10 {
 			status.Status = "Stale"
-			status.Message = fmt.Sprintf("scraper hasn't run for %s", duration.HumanDuration(elapsed))
-		} else if elapsed > schedule && status.Health != HealthUnhealthy {
 			status.Health = HealthWarning
-			status.Status = "Stale"
-			status.Message = fmt.Sprintf("scraper hasn't run for %s", duration.HumanDuration(elapsed))
+			status.Message = fmt.Sprintf("scraper hasn't run for %s", duration.HumanDuration(time.Since(parsedLastRuntime)))
+
+			if time.Since(nextRuntime) > time.Hour {
+				status.Health = HealthUnhealthy
+			}
 		}
 	}
 

pkg/health/health_test.go

@@ -33,6 +33,7 @@ var (
 		"@now-5m":  _now.Add(-time.Minute * 5).Format(RFC3339Micro),
 		"@now-10m": _now.Add(-time.Minute * 10).Format(RFC3339Micro),
 		"@now-15m": _now.Add(-time.Minute * 15).Format(RFC3339Micro),
+		"@now-30m": _now.Add(-time.Minute * 30).Format(RFC3339Micro),
 		"@now-1h":  _now.Add(-time.Hour).Format(RFC3339Micro),
 		"@now-2h":  _now.Add(-time.Hour * 2).Format(RFC3339Micro),
 		"@now-4h":  _now.Add(-time.Hour * 4).Format(RFC3339Micro),

pkg/health/testdata/Kubernetes/MissionControl/scrapeConfig-cron-schedule.yaml

@@ -0,0 +1,23 @@
+apiVersion: configs.flanksource.com/v1
+kind: ScrapeConfig
+metadata:
+  uid: 8e6902db-67bc-4546-a93e-5492112205f9
+  name: ifs-dev-cluster
+  namespace: mission-control-agent
+  finalizers:
+    - scrapeConfig.config.flanksource.com
+  annotations:
+    meta.helm.sh/release-name: dev-kubernetes-bundle
+    meta.helm.sh/release-namespace: mission-control-agent
+    expected-ready: 'true'
+    expected-health: 'unhealthy'
+    expected-message: "scraper hasn't run for 120m"
+  creationTimestamp: '2024-12-04T12:21:48Z'
+spec:
+  schedule: "10 * * * *"
+  kubernetes:
+    - clusterName: ifs-dev-cluster
+status:
+  lastRun:
+    success: 950
+    timestamp: "@now-2h"

pkg/health/testdata/Kubernetes/MissionControl/scrapeConfig-minor-delay.yaml

@@ -41,4 +41,4 @@ spec:
 status:
   lastRun:
     success: 1
-    timestamp: "@now-15m"
+    timestamp: "@now-30m"