add build team availability alerts

rhobs/alerting/data_plane/prometheus.build_service_alerts.yaml

@@ -0,0 +1,23 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: rhtap-build-service-github-app-alerting
+  labels:
+    tenant: rhtap
+spec:
+  groups:
+    - name: build_service_github_app_alerts
+      interval: 1m
+      rules:
+        - alert: GitHubAppFailureAlert
+          expr: absent(konflux_up{service="github", check="build-service"}) == 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: "'konflux_up' availability metric missing for GitHub App in build-service."
+            description: >-
+              The 'konflux_up' availability metric for the GitHub App in the build-service has not been reported for check {{ $labels.check }} on service {{ $labels.service }} for over 5 minutes, indicating a possible service disruption.
+            team: build
+            alert_team_handle: <!subteam^S03DM1RL0TF>
+            runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/build-service/availability_github_app.md

rhobs/alerting/data_plane/prometheus.image_controller_alerts.yaml

@@ -0,0 +1,24 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: rhtap-image-controller-quay-alerting
+  labels:
+    tenant: rhtap
+spec:
+  groups:
+    - name: image_controller_quay_alerts
+      interval: 1m
+      rules:
+        - alert: QuayFailureAlert
+          expr: absent(konflux_up{service="quay", check="image-controller"}) == 1
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            summary: Availability metric 'konflux_up' missing for Quay in image-controller.
+            description: >-
+              The availability metric 'konflux_up' is missing for the Quay service in the image-controller
+              for more than 5 minutes, indicating a potential service failure.
+            team: build
+            alert_team_handle: <!subteam^S03DM1RL0TF>
+            runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/image-controller/availability_quay.md

test/promql/tests/data_plane/github_app_test.yaml

@@ -0,0 +1,31 @@
+rule_files:
+  - prometheus.build_service_alerts.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: konflux_up{service="github", check="build-service"}
+        values: '_x6 1'
+    alert_rule_test:
+      - alertname: GitHubAppFailureAlert
+        eval_time: 4m
+        exp_alerts: []
+      - alertname: GitHubAppFailureAlert
+        eval_time: 5m
+        exp_alerts:
+        - exp_labels:
+            severity: warning
+            check: build-service
+            service: github
+          exp_annotations:
+            summary: "'konflux_up' availability metric missing for GitHub App in build-service."
+            description: >-
+              The 'konflux_up' availability metric for the GitHub App in the build-service has not been reported for check build-service on service github for over 5 minutes, indicating a possible service disruption.
+            team: build
+            alert_team_handle: <!subteam^S03DM1RL0TF>
+            runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/build-service/availability_github_app.md
+      - alertname: GitHubAppFailureAlert
+        eval_time: 7m
+        exp_alerts: []

test/promql/tests/data_plane/quay_failure_test.yaml

@@ -0,0 +1,32 @@
+rule_files:
+  - prometheus.image_controller_alerts.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: konflux_up{service="quay", check="image-controller"}
+        values: '_x6 1'
+    alert_rule_test:
+      - alertname: QuayFailureAlert
+        eval_time: 4m
+        exp_alerts: []
+      - alertname: QuayFailureAlert
+        eval_time: 5m
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              check: image-controller
+              service: quay
+            exp_annotations:
+              summary: Availability metric 'konflux_up' missing for Quay in image-controller.
+              description: >-
+                The availability metric 'konflux_up' is missing for the Quay service in the image-controller
+                for more than 5 minutes, indicating a potential service failure.
+              team: build
+              alert_team_handle: <!subteam^S03DM1RL0TF>
+              runbook_url: https://gitlab.cee.redhat.com/konflux/docs/sop/-/blob/main/image-controller/availability_quay.md
+      - alertname: QuayFailureAlert
+        eval_time: 7m
+        exp_alerts: []