Skip to content

Commit 9cb4a1c

Browse files
authored
feat(ScrapeConfig): LastRuntime check (#164)
* feat(ScrapeConfig): LastRuntime check * chore: change message and status for scrape config scrape delay
1 parent 8dee7c2 commit 9cb4a1c

8 files changed

+236
-11
lines changed

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/cert-manager/cert-manager v1.9.0
88
github.com/flanksource/commons v1.31.2
99
github.com/gobwas/glob v0.2.3
10+
github.com/robfig/cron/v3 v3.0.1
1011
github.com/samber/lo v1.44.0
1112
github.com/stretchr/testify v1.9.0
1213
github.com/yuin/gopher-lua v1.1.0

go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
439439
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
440440
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
441441
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
442+
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
443+
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
442444
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
443445
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
444446
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=

pkg/health/health_flanksource.go

+45-1
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,12 @@ import (
55
"regexp"
66
"strconv"
77
"strings"
8+
"time"
89

10+
"github.com/robfig/cron/v3"
911
"github.com/samber/lo"
1012
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
13+
"k8s.io/apimachinery/pkg/util/duration"
1114
)
1215

1316
var re = regexp.MustCompile(`(?:\((\d+\.?\d*)%\))|(\d+\.?\d*)%`)
@@ -95,7 +98,10 @@ func getScrapeConfigHealth(obj *unstructured.Unstructured) (*HealthStatus, error
9598
h = HealthUnknown
9699
}
97100

98-
status := &HealthStatus{Health: h}
101+
status := &HealthStatus{
102+
Health: h,
103+
Ready: true,
104+
}
99105

100106
if errorCount > 0 {
101107
errorMsgs, _, err := unstructured.NestedStringSlice(obj.Object, "status", "lastRun", "errors")
@@ -108,6 +114,44 @@ func getScrapeConfigHealth(obj *unstructured.Unstructured) (*HealthStatus, error
108114
}
109115
}
110116

117+
if lastRunTime, _, err := unstructured.NestedString(obj.Object, "status", "lastRun", "timestamp"); err != nil {
118+
return nil, err
119+
} else if lastRunTime != "" {
120+
parsedLastRuntime, err := time.Parse(time.RFC3339, lastRunTime)
121+
if err != nil {
122+
return nil, fmt.Errorf("failed to parse lastRun timestamp: %w", err)
123+
}
124+
125+
var schedule time.Duration
126+
if scheduleRaw, _, err := unstructured.NestedString(obj.Object, "spec", "schedule"); err != nil {
127+
return nil, fmt.Errorf("failed to parse scraper schedule: %w", err)
128+
} else if scheduleRaw == "" {
129+
schedule = time.Hour // The default schedule
130+
} else {
131+
parsedSchedule, err := cron.ParseStandard(scheduleRaw)
132+
if err != nil {
133+
return &HealthStatus{
134+
Health: HealthUnhealthy,
135+
Message: fmt.Sprintf("Bad schedule: %s", scheduleRaw),
136+
Ready: true,
137+
}, nil
138+
}
139+
140+
schedule = time.Until(parsedSchedule.Next(time.Now()))
141+
}
142+
143+
elapsed := time.Since(parsedLastRuntime)
144+
if elapsed > schedule*2 {
145+
status.Health = HealthUnhealthy
146+
status.Status = "Stale"
147+
status.Message = fmt.Sprintf("scraper hasn't run for %s", duration.HumanDuration(elapsed))
148+
} else if elapsed > schedule && status.Health != HealthUnhealthy {
149+
status.Health = HealthWarning
150+
status.Status = "Stale"
151+
status.Message = fmt.Sprintf("scraper hasn't run for %s", duration.HumanDuration(elapsed))
152+
}
153+
}
154+
111155
return status, nil
112156
}
113157

pkg/health/health_test.go

+10-10
Original file line numberDiff line numberDiff line change
@@ -30,24 +30,24 @@ var (
3030
defaultOverrides = map[string]string{
3131
"@now": _now.Format(RFC3339Micro),
3232
"@now-1m": _now.Add(-time.Minute * 1).Format(RFC3339Micro),
33-
"@now-10m": _now.Add(-time.Minute * 5).Format(RFC3339Micro),
34-
"@now-15m": _now.Add(-time.Minute * 15).Format(RFC3339Micro),
3533
"@now-5m": _now.Add(-time.Minute * 5).Format(RFC3339Micro),
34+
"@now-10m": _now.Add(-time.Minute * 10).Format(RFC3339Micro),
35+
"@now-15m": _now.Add(-time.Minute * 15).Format(RFC3339Micro),
3636
"@now-1h": _now.Add(-time.Hour).Format(RFC3339Micro),
3737
"@now-2h": _now.Add(-time.Hour * 2).Format(RFC3339Micro),
3838
"@now-4h": _now.Add(-time.Hour * 4).Format(RFC3339Micro),
3939
"@now-8h": _now.Add(-time.Hour * 8).Format(RFC3339Micro),
4040
"@now-1d": _now.Add(-time.Hour * 24).Format(RFC3339Micro),
41-
"@now-5d": _now.Add(-time.Hour * 24).Format(RFC3339Micro),
42-
"@now+10m": _now.Add(time.Minute * 10).Format(RFC3339Micro),
41+
"@now-5d": _now.Add(-time.Hour * 24 * 5).Format(RFC3339Micro),
42+
4343
"@now+5m": _now.Add(time.Minute * 5).Format(RFC3339Micro),
44+
"@now+10m": _now.Add(time.Minute * 10).Format(RFC3339Micro),
4445
"@now+15m": _now.Add(time.Minute * 15).Format(RFC3339Micro),
45-
46-
"@now+1h": _now.Add(time.Hour).Format(RFC3339Micro),
47-
"@now+2h": _now.Add(time.Hour * 2).Format(RFC3339Micro),
48-
"@now+4h": _now.Add(time.Hour * 4).Format(RFC3339Micro),
49-
"@now+8h": _now.Add(time.Hour * 8).Format(RFC3339Micro),
50-
"@now+1d": _now.Add(time.Hour * 24).Format(RFC3339Micro),
46+
"@now+1h": _now.Add(time.Hour).Format(RFC3339Micro),
47+
"@now+2h": _now.Add(time.Hour * 2).Format(RFC3339Micro),
48+
"@now+4h": _now.Add(time.Hour * 4).Format(RFC3339Micro),
49+
"@now+8h": _now.Add(time.Hour * 8).Format(RFC3339Micro),
50+
"@now+1d": _now.Add(time.Hour * 24).Format(RFC3339Micro),
5151
}
5252
)
5353

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
apiVersion: configs.flanksource.com/v1
2+
kind: ScrapeConfig
3+
metadata:
4+
annotations:
5+
expected-ready: 'true'
6+
expected-health: 'warning'
7+
expected-message: 'failed to get server resources'
8+
creationTimestamp: '2024-12-04T12:21:48Z'
9+
finalizers:
10+
- scrapeConfig.config.flanksource.com
11+
generation: 5
12+
name: homelab
13+
namespace: mc
14+
resourceVersion: '145300093'
15+
uid: 139182ab-8637-40df-9f0f-ab9bc419c6f7
16+
spec:
17+
kubernetes:
18+
- clusterName: homelab
19+
exclusions:
20+
kind:
21+
- Secret
22+
- APIService
23+
- ComponentStatus
24+
- PodMetrics
25+
- NodeMetrics
26+
- endpoints.discovery.k8s.io
27+
- endpointslices.discovery.k8s.io
28+
- leases.coordination.k8s.io
29+
- orders.acme.cert-manager.io
30+
labels:
31+
canaries.flanksource.com/check: '*'
32+
canaries.flanksource.com/check-id: '*'
33+
canaries.flanksource.com/generated: 'true'
34+
name:
35+
- junit*
36+
- k6-junit*
37+
- newman-junit*
38+
- playwright-junit-*
39+
namespace: []
40+
schedule: '@every 1h'
41+
status:
42+
lastRun:
43+
error: 1
44+
errors:
45+
- 'failed to get server resources'
46+
success: 1
47+
timestamp: '@now-5m'
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: configs.flanksource.com/v1
2+
kind: ScrapeConfig
3+
metadata:
4+
annotations:
5+
expected-ready: "true"
6+
expected-status: "Stale"
7+
expected-health: "unhealthy"
8+
creationTimestamp: "2024-12-04T12:21:48Z"
9+
finalizers:
10+
- scrapeConfig.config.flanksource.com
11+
generation: 5
12+
name: homelab
13+
namespace: mc
14+
resourceVersion: "145300093"
15+
uid: 139182ab-8637-40df-9f0f-ab9bc419c6f7
16+
spec:
17+
kubernetes:
18+
- clusterName: homelab
19+
exclusions:
20+
kind:
21+
- Secret
22+
- APIService
23+
- ComponentStatus
24+
- PodMetrics
25+
- NodeMetrics
26+
- endpoints.discovery.k8s.io
27+
- endpointslices.discovery.k8s.io
28+
- leases.coordination.k8s.io
29+
- orders.acme.cert-manager.io
30+
labels:
31+
canaries.flanksource.com/check: "*"
32+
canaries.flanksource.com/check-id: "*"
33+
canaries.flanksource.com/generated: "true"
34+
name:
35+
- junit*
36+
- k6-junit*
37+
- newman-junit*
38+
- playwright-junit-*
39+
namespace: []
40+
schedule: "@every 1h"
41+
status:
42+
lastRun:
43+
success: 1
44+
timestamp: "@now-4h"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
apiVersion: configs.flanksource.com/v1
2+
kind: ScrapeConfig
3+
metadata:
4+
annotations:
5+
expected-ready: "true"
6+
expected-status: "Stale"
7+
expected-health: "warning"
8+
creationTimestamp: "2024-12-04T12:21:48Z"
9+
finalizers:
10+
- scrapeConfig.config.flanksource.com
11+
generation: 5
12+
name: homelab
13+
namespace: mc
14+
resourceVersion: "145300093"
15+
uid: 139182ab-8637-40df-9f0f-ab9bc419c6f7
16+
spec:
17+
kubernetes:
18+
- clusterName: homelab
19+
exclusions:
20+
kind:
21+
- Secret
22+
- APIService
23+
- ComponentStatus
24+
- PodMetrics
25+
- NodeMetrics
26+
- endpoints.discovery.k8s.io
27+
- endpointslices.discovery.k8s.io
28+
- leases.coordination.k8s.io
29+
- orders.acme.cert-manager.io
30+
labels:
31+
canaries.flanksource.com/check: "*"
32+
canaries.flanksource.com/check-id: "*"
33+
canaries.flanksource.com/generated: "true"
34+
name:
35+
- junit*
36+
- k6-junit*
37+
- newman-junit*
38+
- playwright-junit-*
39+
namespace: []
40+
schedule: "@every 10m"
41+
status:
42+
lastRun:
43+
success: 1
44+
timestamp: "@now-15m"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
apiVersion: configs.flanksource.com/v1
2+
kind: ScrapeConfig
3+
metadata:
4+
annotations:
5+
expected-ready: "true"
6+
expected-health: "healthy"
7+
creationTimestamp: "2024-12-04T12:21:48Z"
8+
finalizers:
9+
- scrapeConfig.config.flanksource.com
10+
generation: 5
11+
name: homelab
12+
namespace: mc
13+
resourceVersion: "145300093"
14+
uid: 139182ab-8637-40df-9f0f-ab9bc419c6f7
15+
spec:
16+
kubernetes:
17+
- clusterName: homelab
18+
exclusions:
19+
kind:
20+
- Secret
21+
- APIService
22+
- ComponentStatus
23+
- PodMetrics
24+
- NodeMetrics
25+
- endpoints.discovery.k8s.io
26+
- endpointslices.discovery.k8s.io
27+
- leases.coordination.k8s.io
28+
- orders.acme.cert-manager.io
29+
labels:
30+
canaries.flanksource.com/check: "*"
31+
canaries.flanksource.com/check-id: "*"
32+
canaries.flanksource.com/generated: "true"
33+
name:
34+
- junit*
35+
- k6-junit*
36+
- newman-junit*
37+
- playwright-junit-*
38+
namespace: []
39+
schedule: "@every 1h"
40+
status:
41+
lastRun:
42+
success: 1
43+
timestamp: "@now-5m"

0 commit comments

Comments
 (0)