Set distributor.remote-timeout=10s for rulers in jsonnet/helm (#7143)

* Set distributor.remote-timeout to 10s for rulers * Update CHANGELOG.md * Update operations/helm/charts/mimir-distributed/CHANGELOG.md Co-authored-by: Marco Pracucci <[email protected]> * Add comment explaining distributor.remote-timeout=10s * Set -distributor.remote-timeout directly in the ruler deployment args list --------- Co-authored-by: Marco Pracucci <[email protected]>
grafana · Jan 18, 2024 · befb428 · befb428
1 parent b6216b7
commit befb428
Show file tree

Hide file tree

Showing 66 changed files with 77 additions and 0 deletions.
diff --git a/operations/helm/charts/mimir-distributed/CHANGELOG.md b/operations/helm/charts/mimir-distributed/CHANGELOG.md
@@ -38,6 +38,7 @@ Entries should include a reference to the Pull Request that introduced the chang
   * `-store-gateway.sharding-ring.heartbeat-timeout` set to `4m`
   * `-compactor.ring.heartbeat-period` set to `1m`
   * `-compactor.ring.heartbeat-timeout` set to `4m`
+* [CHANGE] Ruler: Set `-distributor.remote-timeout` to 10s in order to accommodate writing large rule results to the ingester. #7143
 * [ENHANCEMENT] Add `jaegerReporterMaxQueueSize` Helm value for all components where configuring `JAEGER_REPORTER_MAX_QUEUE_SIZE` makes sense, and override the Jaeger client's default value of 100 for components expected to generate many trace spans. #7068 #7086
 * [ENHANCEMENT] Rollout-operator: upgraded to v0.10.1. #7125
 * [ENHANCEMENT] Query-frontend: configured `-shutdown-delay`, `-server.grpc.keepalive.max-connection-age` and termination grace period to reduce the likelihood of queries hitting terminated query-frontends. #7129

diff --git a/operations/helm/charts/mimir-distributed/templates/ruler/ruler-dep.yaml b/operations/helm/charts/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -45,6 +45,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           {{- if .Values.ingester.zoneAwareReplication.migration.enabled }}
             {{- if not .Values.ingester.zoneAwareReplication.migration.writePath }}
             - "-ingester.ring.zone-awareness-enabled=false"

diff --git a/.../tests/enterprise-https-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/.../tests/enterprise-https-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
 
             - mountPath: /certs

diff --git a/...ests/gateway-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ests/gateway-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...elm/tests/gateway-nginx-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...elm/tests/gateway-nginx-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/.../tests/graphite-enabled-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/.../tests/graphite-enabled-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...ations/helm/tests/large-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ations/helm/tests/large-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...lm/tests/metamonitoring-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...lm/tests/metamonitoring-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...ns/helm/tests/openshift-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ns/helm/tests/openshift-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -50,6 +50,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...lm/tests/scheduler-name-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...lm/tests/scheduler-name-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...ations/helm/tests/small-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ations/helm/tests/small-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...st-enterprise-configmap-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...st-enterprise-configmap-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -54,6 +54,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...est-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...est-enterprise-k8s-1.25-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...enterprise-legacy-label-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...enterprise-legacy-label-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -50,6 +50,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...m/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...m/tests/test-enterprise-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...helm/tests/test-ingress-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...helm/tests/test-ingress-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...t-oss-logical-multizone-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...t-oss-logical-multizone-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...ests/test-oss-multizone-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ests/test-oss-multizone-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...logy-spread-constraints-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...logy-spread-constraints-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -53,6 +53,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/...ons/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/...ons/helm/tests/test-oss-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -54,6 +54,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/.../tests/test-vault-agent-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml b/.../tests/test-vault-agent-values-generated/mimir-distributed/templates/ruler/ruler-dep.yaml
@@ -60,6 +60,7 @@ spec:
             - "-target=ruler"
             - "-config.expand-env=true"
             - "-config.file=/etc/mimir/mimir.yaml"
+            - "-distributor.remote-timeout=10s"
           volumeMounts:
             - name: config
               mountPath: /etc/mimir

diff --git a/operations/mimir-tests/test-all-components-generated.yaml b/operations/mimir-tests/test-all-components-generated.yaml
@@ -840,6 +840,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-all-components-with-tsdb-head-early-compaction-generated.yaml b/operations/mimir-tests/test-all-components-with-tsdb-head-early-compaction-generated.yaml
@@ -840,6 +840,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-all-components-without-chunk-streaming-generated.yaml b/operations/mimir-tests/test-all-components-without-chunk-streaming-generated.yaml
@@ -841,6 +841,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-automated-downscale-generated.yaml b/operations/mimir-tests/test-automated-downscale-generated.yaml
@@ -1063,6 +1063,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml b/operations/mimir-tests/test-autoscaling-custom-target-utilization-generated.yaml
@@ -953,6 +953,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-autoscaling-generated.yaml b/operations/mimir-tests/test-autoscaling-generated.yaml
@@ -953,6 +953,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-consul-generated.yaml b/operations/mimir-tests/test-consul-generated.yaml
@@ -1222,6 +1222,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=

diff --git a/operations/mimir-tests/test-consul-multi-zone-generated.yaml b/operations/mimir-tests/test-consul-multi-zone-generated.yaml
@@ -1433,6 +1433,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=

diff --git a/operations/mimir-tests/test-deployment-mode-migration-generated.yaml b/operations/mimir-tests/test-deployment-mode-migration-generated.yaml
@@ -1464,6 +1464,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3
@@ -2396,6 +2397,7 @@ spec:
         - -distributor.health-check-ingesters=true
         - -distributor.ingestion-burst-size=200000
         - -distributor.ingestion-rate-limit=10000
+        - -distributor.remote-timeout=10s
         - -ingester.max-global-metadata-per-metric=10
         - -ingester.max-global-metadata-per-user=30000
         - -ingester.max-global-series-per-user=150000
@@ -2588,6 +2590,7 @@ spec:
         - -distributor.health-check-ingesters=true
         - -distributor.ingestion-burst-size=200000
         - -distributor.ingestion-rate-limit=10000
+        - -distributor.remote-timeout=10s
         - -ingester.max-global-metadata-per-metric=10
         - -ingester.max-global-metadata-per-user=30000
         - -ingester.max-global-series-per-user=150000
@@ -2780,6 +2783,7 @@ spec:
         - -distributor.health-check-ingesters=true
         - -distributor.ingestion-burst-size=200000
         - -distributor.ingestion-rate-limit=10000
+        - -distributor.remote-timeout=10s
         - -ingester.max-global-metadata-per-metric=10
         - -ingester.max-global-metadata-per-user=30000
         - -ingester.max-global-series-per-user=150000

diff --git a/operations/mimir-tests/test-env-vars-generated.yaml b/operations/mimir-tests/test-env-vars-generated.yaml
@@ -840,6 +840,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-extra-runtime-config-generated.yaml b/operations/mimir-tests/test-extra-runtime-config-generated.yaml
@@ -868,6 +868,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-helm-parity-generated.yaml b/operations/mimir-tests/test-helm-parity-generated.yaml
@@ -942,6 +942,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=example-blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-cluster-label-migration-step-0-before-generated.yaml b/operations/mimir-tests/test-memberlist-cluster-label-migration-step-0-before-generated.yaml
@@ -840,6 +840,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-cluster-label-migration-step-1-generated.yaml b/operations/mimir-tests/test-memberlist-cluster-label-migration-step-1-generated.yaml
@@ -842,6 +842,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-cluster-label-migration-step-2-generated.yaml b/operations/mimir-tests/test-memberlist-cluster-label-migration-step-2-generated.yaml
@@ -844,6 +844,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-cluster-label-migration-step-3-generated.yaml b/operations/mimir-tests/test-memberlist-cluster-label-migration-step-3-generated.yaml
@@ -842,6 +842,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-migration-step-0-before-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-0-before-generated.yaml
@@ -1222,6 +1222,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=

diff --git a/operations/mimir-tests/test-memberlist-migration-step-1-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-1-generated.yaml
@@ -1278,6 +1278,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.multi.primary=consul

diff --git a/operations/mimir-tests/test-memberlist-migration-step-2-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-2-generated.yaml
@@ -1278,6 +1278,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.multi.primary=consul

diff --git a/operations/mimir-tests/test-memberlist-migration-step-3-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-3-generated.yaml
@@ -1278,6 +1278,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.multi.primary=consul

diff --git a/operations/mimir-tests/test-memberlist-migration-step-4-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-4-generated.yaml
@@ -1278,6 +1278,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.consul.hostname=consul.default.svc.cluster.local.:8500
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.multi.primary=consul

diff --git a/operations/mimir-tests/test-memberlist-migration-step-5-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-5-generated.yaml
@@ -843,6 +843,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memberlist-migration-step-6-final-generated.yaml b/operations/mimir-tests/test-memberlist-migration-step-6-final-generated.yaml
@@ -840,6 +840,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-memcached-mtls-generated.yaml b/operations/mimir-tests/test-memcached-mtls-generated.yaml
@@ -894,6 +894,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3

diff --git a/operations/mimir-tests/test-multi-zone-generated.yaml b/operations/mimir-tests/test-multi-zone-generated.yaml
@@ -1063,6 +1063,7 @@ spec:
         - -blocks-storage.gcs.bucket-name=blocks-bucket
         - -common.storage.backend=gcs
         - -distributor.health-check-ingesters=true
+        - -distributor.remote-timeout=10s
         - -ingester.ring.heartbeat-timeout=10m
         - -ingester.ring.prefix=
         - -ingester.ring.replication-factor=3