feat: notification health

Author: adityathebe

pkg/health/health_fixtures_test.go

@@ -16,7 +16,7 @@ func TestFixtures(t *testing.T) {
 	}
 
 	for _, file := range files {
-		// if file != "testdata/Kubernetes/MissionControl/scrapeConfig-minor-delay.yaml" {
+		// if file != "testdata/Kubernetes/MissionControl/notification-with-error.yaml" {
 		// 	continue
 		// }
 

pkg/health/health_flanksource.go

@@ -176,25 +176,53 @@ func getNotificationHealth(obj *unstructured.Unstructured) (*HealthStatus, error
 		return nil, err
 	}
 
-	var h Health = HealthUnknown
+	status := &HealthStatus{
+		Health: HealthUnknown,
+		Ready:  true,
+	}
+
+	if errorMessage != "" {
+		status.Message = errorMessage
+		status.Health = HealthUnhealthy
+		status.Ready = false
+		return status, nil
+	}
+
 	if sentCount > 0 {
-		h = HealthHealthy
+		status.Health = HealthHealthy
 		if failedCount > 0 || pendingCount > 0 {
-			h = HealthWarning
+			status.Health = HealthWarning
 		}
 	} else {
 		if pendingCount > 0 {
-			h = HealthWarning
+			status.Health = HealthWarning
 		}
 		if failedCount > 0 {
-			h = HealthUnhealthy
+			status.Health = HealthUnhealthy
 		}
 	}
 
-	status := &HealthStatus{Health: h}
+	// Check lastFailed timestamp
+	lastFailedTime, found, err := unstructured.NestedString(obj.Object, "status", "lastFailed")
+	if err != nil {
+		return nil, err
+	}
+
+	if found && lastFailedTime != "" {
+		parsedLastFailedTime, err := time.Parse(time.RFC3339, lastFailedTime)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse lastFailed timestamp: %w", err)
+		}
 
-	if errorMessage != "" {
-		status.Message = errorMessage
+		timeSinceLastFailure := time.Since(parsedLastFailedTime)
+
+		if timeSinceLastFailure <= 12*time.Hour {
+			status.Health = HealthWarning
+			status.Message = fmt.Sprintf("Failed %s ago", duration.HumanDuration(timeSinceLastFailure))
+			if timeSinceLastFailure <= time.Hour {
+				status.Health = HealthUnhealthy
+			}
+		}
 	}
 
 	return status, nil

pkg/health/testdata/Kubernetes/MissionControl/notification-last-failed-a-while-ago.yaml

@@ -0,0 +1,44 @@
+apiVersion: mission-control.flanksource.com/v1
+kind: Notification
+metadata:
+  annotations:
+    meta.helm.sh/release-name: playbooks-ai
+    meta.helm.sh/release-namespace: mc
+    expected-ready: 'true'
+    expected-health: 'warning'
+    expected-message: "Failed 120m ago"
+  creationTimestamp: "2025-03-11T07:12:44Z"
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: notify-recommender-playbook
+  namespace: mc
+  resourceVersion: "147557980"
+  uid: a37a3b53-951d-40f3-a65a-b540950ceb0f
+spec:
+  events:
+  - config.unhealthy
+  - config.warning
+  fallback:
+    connection: connection://mc/flanksource-slack
+  filter: tags.cluster == 'homelab'
+  inhibitions:
+  - direction: incoming
+    from: Kubernetes::Pod
+    to:
+    - Kubernetes::Deployment
+    - Kubernetes::ReplicaSet
+    - Kubernetes::DaemonSet
+    - Kubernetes::StatefulSet
+  - direction: outgoing
+    from: Kubernetes::Node
+    to:
+    - Kubernetes::Pod
+  repeatInterval: 1d
+  to:
+    playbook: mc/recommend-playbook
+  waitFor: 20s
+status:
+  failed: 1
+  lastSent: "2025-03-14T04:37:50Z"
+  lastFailed: "@now-2h"
+  sent: 4

pkg/health/testdata/Kubernetes/MissionControl/notification-last-failed-recently.yaml

@@ -0,0 +1,44 @@
+apiVersion: mission-control.flanksource.com/v1
+kind: Notification
+metadata:
+  annotations:
+    meta.helm.sh/release-name: playbooks-ai
+    meta.helm.sh/release-namespace: mc
+    expected-ready: 'true'
+    expected-health: 'unhealthy'
+    expected-message: "Failed 30m ago"
+  creationTimestamp: "2025-03-11T07:12:44Z"
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: notify-recommender-playbook
+  namespace: mc
+  resourceVersion: "147557980"
+  uid: a37a3b53-951d-40f3-a65a-b540950ceb0f
+spec:
+  events:
+  - config.unhealthy
+  - config.warning
+  fallback:
+    connection: connection://mc/flanksource-slack
+  filter: tags.cluster == 'homelab'
+  inhibitions:
+  - direction: incoming
+    from: Kubernetes::Pod
+    to:
+    - Kubernetes::Deployment
+    - Kubernetes::ReplicaSet
+    - Kubernetes::DaemonSet
+    - Kubernetes::StatefulSet
+  - direction: outgoing
+    from: Kubernetes::Node
+    to:
+    - Kubernetes::Pod
+  repeatInterval: 1d
+  to:
+    playbook: mc/recommend-playbook
+  waitFor: 20s
+status:
+  failed: 1
+  lastSent: "2025-03-14T04:37:50Z"
+  lastFailed: "@now-30m"
+  sent: 4

pkg/health/testdata/Kubernetes/MissionControl/notification-with-error.yaml

@@ -0,0 +1,45 @@
+apiVersion: mission-control.flanksource.com/v1
+kind: Notification
+metadata:
+  annotations:
+    meta.helm.sh/release-name: playbooks-ai
+    meta.helm.sh/release-namespace: mc
+    expected-ready: 'false'
+    expected-health: 'unhealthy'
+    expected-message: "tcp: lookup slack.com: no such host"
+  creationTimestamp: "2025-03-11T07:12:44Z"
+  labels:
+    app.kubernetes.io/managed-by: Helm
+  name: notify-recommender-playbook
+  namespace: mc
+  resourceVersion: "147557980"
+  uid: a37a3b53-951d-40f3-a65a-b540950ceb0f
+spec:
+  events:
+  - config.unhealthy
+  - config.warning
+  fallback:
+    connection: connection://mc/flanksource-slack
+  filter: tags.cluster == 'homelab'
+  inhibitions:
+  - direction: incoming
+    from: Kubernetes::Pod
+    to:
+    - Kubernetes::Deployment
+    - Kubernetes::ReplicaSet
+    - Kubernetes::DaemonSet
+    - Kubernetes::StatefulSet
+  - direction: outgoing
+    from: Kubernetes::Node
+    to:
+    - Kubernetes::Pod
+  repeatInterval: 1d
+  to:
+    playbook: mc/recommend-playbook
+  waitFor: 20s
+status:
+  failed: 1
+  lastSent: "2025-03-14T04:37:50Z"
+  lastFailed: "@now-2h"
+  sent: 4
+  error: "tcp: lookup slack.com: no such host"