Ingester client: configure cluster validation label via common config…

…urations

Signed-off-by: Yuri Nikolic <[email protected]>

cmd/mimir/help-all.txt.tmpl

@@ -191,6 +191,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -alertmanager.alertmanager-client.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -alertmanager.alertmanager-client.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -alertmanager.alertmanager-client.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -alertmanager.alertmanager-client.connect-backoff-max-delay duration
@@ -987,6 +989,8 @@ Usage of ./cmd/mimir/mimir:
     	Maximum number of CPUs that can simultaneously processes WAL replay. If it is set to 0, then each TSDB is replayed with a concurrency equal to the number of CPU cores available on the machine.
   -blocks-storage.tsdb.wal-segment-size-bytes int
     	TSDB WAL segments files max size (bytes). (default 134217728)
+  -common.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -common.storage.azure.account-key string
     	Azure storage account key. If unset, Azure managed identities will be used for authentication instead.
   -common.storage.azure.account-name string
@@ -1591,50 +1595,6 @@ Usage of ./cmd/mimir/mimir:
     	After what time a series is considered to be inactive. (default 20m0s)
   -ingester.active-series-metrics-update-period duration
     	How often to update active series metrics. (default 1m0s)
-  -ingester.client.backoff-max-period duration
-    	Maximum delay when backing off. (default 10s)
-  -ingester.client.backoff-min-period duration
-    	Minimum delay when backing off. (default 100ms)
-  -ingester.client.backoff-on-ratelimits
-    	Enable backoff and retry when we hit rate limits.
-  -ingester.client.backoff-retries int
-    	Number of times to backoff and retry before failing. (default 10)
-  -ingester.client.connect-backoff-base-delay duration
-    	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
-  -ingester.client.connect-backoff-max-delay duration
-    	Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0. (default 5s)
-  -ingester.client.connect-timeout duration
-    	The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff. (default 5s)
-  -ingester.client.grpc-client-rate-limit float
-    	Rate limit for gRPC client; 0 means disabled.
-  -ingester.client.grpc-client-rate-limit-burst int
-    	Rate limit burst for gRPC client.
-  -ingester.client.grpc-compression string
-    	Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2' and '' (disable compression)
-  -ingester.client.grpc-max-recv-msg-size int
-    	gRPC client max receive message size (bytes). (default 104857600)
-  -ingester.client.grpc-max-send-msg-size int
-    	gRPC client max send message size (bytes). (default 104857600)
-  -ingester.client.initial-connection-window-size value
-    	[experimental] Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
-  -ingester.client.initial-stream-window-size value
-    	[experimental] Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
-  -ingester.client.tls-ca-path string
-    	Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
-  -ingester.client.tls-cert-path string
-    	Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
-  -ingester.client.tls-cipher-suites string
-    	Override the default cipher suite list (separated by commas).
-  -ingester.client.tls-enabled
-    	Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.
-  -ingester.client.tls-insecure-skip-verify
-    	Skip validating server certificate.
-  -ingester.client.tls-key-path string
-    	Path to the key for the client certificate. Also requires the client certificate to be configured.
-  -ingester.client.tls-min-version string
-    	Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13
-  -ingester.client.tls-server-name string
-    	Override the expected name on the server certificate.
   -ingester.error-sample-rate int
     	Each error will be logged once in this many times. Use 0 to log all of them. (default 10)
   -ingester.ignore-ooo-exemplars
@@ -1919,6 +1879,52 @@ Usage of ./cmd/mimir/mimir:
     	[experimental] Period with which to update the per-tenant TSDB configuration. (default 15s)
   -ingester.use-ingester-owned-series-for-limits
     	[experimental] When enabled, only series currently owned by ingester according to the ring are used when checking user per-tenant series limit.
+  -ingester_client.backoff-max-period duration
+    	Maximum delay when backing off. (default 10s)
+  -ingester_client.backoff-min-period duration
+    	Minimum delay when backing off. (default 100ms)
+  -ingester_client.backoff-on-ratelimits
+    	Enable backoff and retry when we hit rate limits.
+  -ingester_client.backoff-retries int
+    	Number of times to backoff and retry before failing. (default 10)
+  -ingester_client.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
+  -ingester_client.connect-backoff-base-delay duration
+    	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
+  -ingester_client.connect-backoff-max-delay duration
+    	Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0. (default 5s)
+  -ingester_client.connect-timeout duration
+    	The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff. (default 5s)
+  -ingester_client.grpc-client-rate-limit float
+    	Rate limit for gRPC client; 0 means disabled.
+  -ingester_client.grpc-client-rate-limit-burst int
+    	Rate limit burst for gRPC client.
+  -ingester_client.grpc-compression string
+    	Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2' and '' (disable compression)
+  -ingester_client.grpc-max-recv-msg-size int
+    	gRPC client max receive message size (bytes). (default 104857600)
+  -ingester_client.grpc-max-send-msg-size int
+    	gRPC client max send message size (bytes). (default 104857600)
+  -ingester_client.initial-connection-window-size value
+    	[experimental] Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
+  -ingester_client.initial-stream-window-size value
+    	[experimental] Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B)
+  -ingester_client.tls-ca-path string
+    	Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.
+  -ingester_client.tls-cert-path string
+    	Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.
+  -ingester_client.tls-cipher-suites string
+    	Override the default cipher suite list (separated by commas).
+  -ingester_client.tls-enabled
+    	Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.
+  -ingester_client.tls-insecure-skip-verify
+    	Skip validating server certificate.
+  -ingester_client.tls-key-path string
+    	Path to the key for the client certificate. Also requires the client certificate to be configured.
+  -ingester_client.tls-min-version string
+    	Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13
+  -ingester_client.tls-server-name string
+    	Override the expected name on the server certificate.
   -log.format string
     	Output log messages in the given format. Valid formats: [logfmt, json] (default "logfmt")
   -log.level value
@@ -2115,6 +2121,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -querier.frontend-client.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -querier.frontend-client.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -querier.frontend-client.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -querier.frontend-client.connect-backoff-max-delay duration
@@ -2223,6 +2231,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -querier.scheduler-client.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -querier.scheduler-client.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -querier.scheduler-client.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -querier.scheduler-client.connect-backoff-max-delay duration
@@ -2305,6 +2315,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -query-frontend.grpc-client-config.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -query-frontend.grpc-client-config.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -query-frontend.grpc-client-config.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -query-frontend.grpc-client-config.connect-backoff-max-delay duration
@@ -2521,6 +2533,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -query-scheduler.grpc-client-config.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -query-scheduler.grpc-client-config.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -query-scheduler.grpc-client-config.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -query-scheduler.grpc-client-config.connect-backoff-max-delay duration
@@ -2955,6 +2969,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -ruler.client.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -ruler.client.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -ruler.client.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -ruler.client.connect-backoff-max-delay duration
@@ -3043,6 +3059,8 @@ Usage of ./cmd/mimir/mimir:
     	Enable backoff and retry when we hit rate limits.
   -ruler.query-frontend.grpc-client-config.backoff-retries int
     	Number of times to backoff and retry before failing. (default 10)
+  -ruler.query-frontend.grpc-client-config.cluster-validation-label string
+    	[experimental] Optionally define gRPC client's cluster validation label.
   -ruler.query-frontend.grpc-client-config.connect-backoff-base-delay duration
     	Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s)
   -ruler.query-frontend.grpc-client-config.connect-backoff-max-delay duration

docs/sources/mimir/configure/configuration-parameters/index.md

@@ -510,6 +510,10 @@ storage:
   # system as object storage backend.
   # The CLI flags prefix for this block configuration is: common.storage
   [filesystem: <filesystem_storage_backend>]
+
+# (experimental) Optionally define gRPC client's cluster validation label.
+# CLI flag: -common.cluster-validation-label
+[cluster_validation_label: <string> | default = ""]
 ```
 
 ### server
@@ -2669,6 +2673,10 @@ alertmanager_client:
   # CLI flag: -alertmanager.alertmanager-client.connect-backoff-max-delay
   [connect_backoff_max_delay: <duration> | default = 5s]
 
+  # (experimental) Optionally define gRPC client's cluster validation label.
+  # CLI flag: -alertmanager.alertmanager-client.cluster-validation-label
+  [cluster_validation_label: <string> | default = ""]
+
 # (advanced) The interval between persisting the current alertmanager state
 # (notification log and silences) to object storage. This is only used when
 # sharding is enabled. This state is read when all replicas for a shard can not
@@ -2776,15 +2784,15 @@ The `ingester_client` block configures how the distributors connect to the inges
 ```yaml
 # Configures the gRPC client used to communicate with ingesters from
 # distributors, queriers and rulers.
-# The CLI flags prefix for this block configuration is: ingester.client
+# The CLI flags prefix for this block configuration is: ingester_client
 [grpc_client_config: <grpc_client>]
 ```
 
 ### grpc_client
 
 The `grpc_client` block configures the gRPC client used to communicate between two Mimir components. The supported CLI flags `<prefix>` used to reference this configuration block are:
 
-- `ingester.client`
+- `ingester_client`
 - `querier.frontend-client`
 - `querier.scheduler-client`
 - `query-frontend.grpc-client-config`
@@ -2927,6 +2935,10 @@ backoff_config:
 # if ConnectTimeout > 0.
 # CLI flag: -<prefix>.connect-backoff-max-delay
 [connect_backoff_max_delay: <duration> | default = 5s]
+
+# (experimental) Optionally define gRPC client's cluster validation label.
+# CLI flag: -<prefix>.cluster-validation-label
+[cluster_validation_label: <string> | default = ""]
 ```
 
 ### frontend_worker

go.mod

@@ -22,7 +22,7 @@ require (
 	github.com/golang/snappy v0.0.4
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.1
-	github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237
+	github.com/grafana/dskit v0.0.0-20250303214858-d23654211757
 	github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc
 	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/influxdata/influxdb/v2 v2.7.11

go.sum

@@ -1272,8 +1272,8 @@ github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85T
 github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4=
 github.com/grafana/alerting v0.0.0-20250225150117-15e285d78df2 h1:kESrzm0FcRVLmGIQCgl1MCwDGLH4sLzWphr7mcFdbfI=
 github.com/grafana/alerting v0.0.0-20250225150117-15e285d78df2/go.mod h1:hdGB3dSl8Ma9Rjo2YiAEAjMkZ5HiNJbNDqRKDefRZrM=
-github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237 h1:VZagYtPcmjgazfPAuWN7lER6mprG20r51+1eYPpATkw=
-github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237/go.mod h1:cu2zIOHhAgRaIDuECsERftSp1l7KHq1aX1jgihQCu0c=
+github.com/grafana/dskit v0.0.0-20250303214858-d23654211757 h1:nAd6h3RfteaAMeTO4cJLcPQGm1X7uYxv5oAhZICkBNw=
+github.com/grafana/dskit v0.0.0-20250303214858-d23654211757/go.mod h1:cu2zIOHhAgRaIDuECsERftSp1l7KHq1aX1jgihQCu0c=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM=
 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc/go.mod h1:JVmqPBe8A/pZWwRoJW5ZjyALeY5OXMzPl7LrVXOdZAI=
 github.com/grafana/franz-go v0.0.0-20241009100846-782ba1442937 h1:fwwnG/NcygoS6XbAaEyK2QzMXI/BZIEJvQ3CD+7XZm8=

integration/ingester_test.go

@@ -849,6 +849,7 @@ func TestInvalidClusterValidationLabel(t *testing.T) {
 			baseFlags := map[string]string{
 				"-distributor.ingestion-tenant-shard-size": "0",
 				"-ingester.ring.heartbeat-period":          "1s",
+				"-common.cluster-validation-label":         testCase.distributorClusterLabel,
 			}
 
 			flags := mergeFlags(
@@ -857,13 +858,6 @@ func TestInvalidClusterValidationLabel(t *testing.T) {
 				baseFlags,
 			)
 
-			distributorFlags := mergeFlags(
-				flags,
-				map[string]string{
-					"-server.cluster-validation.label": testCase.distributorClusterLabel,
-				},
-			)
-
 			ingesterFlags := mergeFlags(
 				flags,
 				map[string]string{
@@ -879,7 +873,7 @@ func TestInvalidClusterValidationLabel(t *testing.T) {
 			require.NoError(t, s.StartAndWaitReady(consul, minio))
 
 			// Start Mimir components.
-			distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), distributorFlags)
+			distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flags)
 			ingester := e2emimir.NewIngester("ingester", consul.NetworkHTTPEndpoint(), ingesterFlags)
 			require.NoError(t, s.StartAndWaitReady(distributor, ingester))
 

pkg/ingester/client/client.go

@@ -38,8 +38,8 @@ func MakeIngesterClient(inst ring.InstanceDesc, cfg Config, metrics *Metrics, lo
 	reportGRPCStatusesOptions := []middleware.InstrumentationOption{middleware.ReportGRPCStatusOption}
 	unary, stream := grpcclient.Instrument(metrics.requestDuration, reportGRPCStatusesOptions...)
 	unary = append(unary, querierapi.ReadConsistencyClientUnaryInterceptor)
-	if cfg.ClusterValidationLabel != "" {
-		unary = append(unary, middleware.ClusterUnaryClientInterceptor(cfg.ClusterValidationLabel, metrics.invalidClusterVerificationLabels, logger))
+	if cfg.GRPCClientConfig.ClusterValidationLabel != "" {
+		unary = append(unary, middleware.ClusterUnaryClientInterceptor(cfg.GRPCClientConfig.ClusterValidationLabel, metrics.invalidClusterVerificationLabels, logger))
 	}
 	stream = append(stream, querierapi.ReadConsistencyClientStreamInterceptor)
 
@@ -70,8 +70,7 @@ func (c *closableHealthAndIngesterClient) Close() error {
 
 // Config is the configuration struct for the ingester client
 type Config struct {
-	GRPCClientConfig       grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with ingesters from distributors, queriers and rulers."`
-	ClusterValidationLabel string            `yaml:"-"`
+	GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with ingesters from distributors, queriers and rulers."`
 }
 
 // RegisterFlags registers configuration settings used by the ingester client config.