Merge pull request #3 from seanankenbruck/rollout-operator-config

Additional changes to resolve outstanding comments
grafana · Mar 4, 2025 · d415a93 · d415a93
2 parents 096dc67 + 1ac7705
commit d415a93
Show file tree

Hide file tree

Showing 26 changed files with 334 additions and 6 deletions.
diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -192,7 +192,7 @@ Entries should include a reference to the Pull Request that introduced the chang
 * [BUGFIX] Let the unified gateway/nginx config listen on IPv6 as well. Followup to #5948. #6204
 * [BUGFIX] Quote `checksum/config` when using external config. This allows setting `externalConfigVersion` to numeric values. #6407
 * [BUGFIX] Update memcached-exporter to 0.14.1 due to CVE-2023-39325. #6861
-  
+
 ## 5.1.4
 
 * [BUGFIX] Update memcached-exporter to 0.14.1 due to CVE-2023-39325.

diff --git a/operations/helm/charts/mimir-distributed/templates/validate.yaml b/operations/helm/charts/mimir-distributed/templates/validate.yaml
@@ -268,3 +268,9 @@
 {{- if and (not .Values.enterprise.enabled) .Values.federation_frontend.enabled -}}
 {{- fail "You have federation_frontend.enabled=true, but enterprise.enabled=false. Federation Frontend is an enterprise-only feature and requires enterprise mode to be enabled." }}
 {{- end -}}
+
+{{- range .Values.ingester.zoneAwareReplication.zones }}
+  {{- if and (eq .noDownscale true) (eq .prepareDownscale true) }}
+    {{- fail (printf "For ingester zone '%s': noDownscale and prepareDownscale cannot be used simultaneously." .name) -}}
+  {{- end }}
+{{- end }}
diff --git a/operations/helm/charts/mimir-distributed/values.yaml b/operations/helm/charts/mimir-distributed/values.yaml
@@ -1180,7 +1180,7 @@ ingester:
         # If undefined or set to null (the default), no Annotation is set
         downscaleLeader: null
         # -- prepareDownscale adds labels and annotations for the rollout-operator to prepare downscaling: https://github.com/grafana/rollout-operator#how-scaling-up-and-down-works
-        prepareDownscale: false
+        prepareDownscale: true
       # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
       - name: zone-b
         # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
@@ -1199,9 +1199,9 @@ ingester:
         # -- downscaleLeader is an Annotation used by the rollout-operator
         # If defined, downscaleLeader: <downscaleLeader>
         # If undefined or set to null (the default), no Annotation is set
-        downscaleLeader: null
+        downscaleLeader: mimir-ingester-zone-a
         # -- prepareDownscale adds labels and annotations for the rollout-operator to prepare downscaling: https://github.com/grafana/rollout-operator#how-scaling-up-and-down-works
-        prepareDownscale: false
+        prepareDownscale: true
       # -- Name of the zone, used in labels and selectors. Must follow Kubernetes naming restrictions: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
       - name: zone-c
         # -- nodeselector to restrict where pods of this zone can be placed. E.g.:
@@ -1220,9 +1220,9 @@ ingester:
         # -- downscaleLeader is an Annotation used by the rollout-operator
         # If defined, downscaleLeader: <downscaleLeader>
         # If undefined or set to null (the default), no Annotation is set
-        downscaleLeader: null
+        downscaleLeader: mimir-ingester-zone-b
         # -- prepareDownscale adds labels and annotations for the rollout-operator to prepare downscaling: https://github.com/grafana/rollout-operator#how-scaling-up-and-down-works
-        prepareDownscale: false
+        prepareDownscale: true
 
 overrides_exporter:
   enabled: true

diff --git a/...ise-https-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...ise-https-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -163,8 +167,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -313,8 +322,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...nterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...nterprise-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -154,8 +158,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -295,8 +304,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...way-nginx-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...way-nginx-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -149,8 +153,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -285,8 +294,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...ng-global-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...ng-global-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -149,8 +153,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -285,8 +294,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...onitoring-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -149,8 +153,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -285,8 +294,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...toscaling-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...toscaling-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -149,8 +153,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -285,8 +294,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...sts/large-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...sts/large-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -165,8 +169,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -317,8 +326,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel

diff --git a/...openshift-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml b/...openshift-values-generated/mimir-distributed/templates/ingester/ingester-statefulset.yaml
@@ -13,8 +13,12 @@ metadata:
     name: "ingester-zone-a"
     rollout-group: ingester
     zone: zone-a
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -151,8 +155,13 @@ metadata:
     name: "ingester-zone-b"
     rollout-group: ingester
     zone: zone-b
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel
@@ -289,8 +298,13 @@ metadata:
     name: "ingester-zone-c"
     rollout-group: ingester
     zone: zone-c
+    grafana.com/prepare-downscale: true
+    grafana.com/min-time-between-zones-downscale: 12h
   annotations:
     rollout-max-unavailable: "50"
+    grafana.com/prepare-downscale-http-path: ingester/prepare-shutdown
+    grafana.com/prepare-downscale-http-port: 8080
+    grafana.com/rollout-downscale-leader: $rolloutZone.downscaleLeader
   namespace: "citestns"
 spec:
   podManagementPolicy: Parallel