From 9dd57ddfa7b126b1e28e4730146f7c0b07d6907b Mon Sep 17 00:00:00 2001 From: Yuri Nikolic Date: Tue, 4 Mar 2025 00:34:51 +0100 Subject: [PATCH] Ingester client: configure cluster validation label via common configurations Signed-off-by: Yuri Nikolic --- cmd/mimir/config-descriptor.json | 143 +++++++++++++++--- cmd/mimir/help-all.txt.tmpl | 106 +++++++------ .../configuration-parameters/index.md | 16 +- go.mod | 2 +- go.sum | 4 +- integration/ingester_test.go | 10 +- pkg/ingester/client/client.go | 7 +- pkg/mimir/mimir.go | 34 ++++- pkg/mimir/mimir_config_test.go | 9 +- pkg/mimir/modules.go | 1 - .../grafana/dskit/grpcclient/grpcclient.go | 3 + vendor/modules.txt | 2 +- 12 files changed, 247 insertions(+), 90 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index d561e439359..14c96aa54e6 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -2156,7 +2156,7 @@ "desc": "gRPC client max receive message size (bytes).", "fieldValue": null, "fieldDefaultValue": 104857600, - "fieldFlag": "ingester.client.grpc-max-recv-msg-size", + "fieldFlag": "ingester_client.grpc-max-recv-msg-size", "fieldType": "int", "fieldCategory": "advanced" }, @@ -2167,7 +2167,7 @@ "desc": "gRPC client max send message size (bytes).", "fieldValue": null, "fieldDefaultValue": 104857600, - "fieldFlag": "ingester.client.grpc-max-send-msg-size", + "fieldFlag": "ingester_client.grpc-max-send-msg-size", "fieldType": "int", "fieldCategory": "advanced" }, @@ -2178,7 +2178,7 @@ "desc": "Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2' and '' (disable compression)", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.grpc-compression", + "fieldFlag": "ingester_client.grpc-compression", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2189,7 +2189,7 @@ "desc": "Rate limit for gRPC client; 0 means disabled.", "fieldValue": null, "fieldDefaultValue": 0, - "fieldFlag": "ingester.client.grpc-client-rate-limit", + "fieldFlag": "ingester_client.grpc-client-rate-limit", "fieldType": "float", "fieldCategory": "advanced" }, @@ -2200,7 +2200,7 @@ "desc": "Rate limit burst for gRPC client.", "fieldValue": null, "fieldDefaultValue": 0, - "fieldFlag": "ingester.client.grpc-client-rate-limit-burst", + "fieldFlag": "ingester_client.grpc-client-rate-limit-burst", "fieldType": "int", "fieldCategory": "advanced" }, @@ -2211,7 +2211,7 @@ "desc": "Enable backoff and retry when we hit rate limits.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ingester.client.backoff-on-ratelimits", + "fieldFlag": "ingester_client.backoff-on-ratelimits", "fieldType": "boolean", "fieldCategory": "advanced" }, @@ -2228,7 +2228,7 @@ "desc": "Minimum delay when backing off.", "fieldValue": null, "fieldDefaultValue": 100000000, - "fieldFlag": "ingester.client.backoff-min-period", + "fieldFlag": "ingester_client.backoff-min-period", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -2239,7 +2239,7 @@ "desc": "Maximum delay when backing off.", "fieldValue": null, "fieldDefaultValue": 10000000000, - "fieldFlag": "ingester.client.backoff-max-period", + "fieldFlag": "ingester_client.backoff-max-period", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -2250,7 +2250,7 @@ "desc": "Number of times to backoff and retry before failing.", "fieldValue": null, "fieldDefaultValue": 10, - "fieldFlag": "ingester.client.backoff-retries", + "fieldFlag": "ingester_client.backoff-retries", "fieldType": "int", "fieldCategory": "advanced" } @@ -2265,7 +2265,7 @@ "desc": "Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator.", "fieldValue": null, "fieldDefaultValue": null, - "fieldFlag": "ingester.client.initial-stream-window-size", + "fieldFlag": "ingester_client.initial-stream-window-size", "fieldType": "int", "fieldCategory": "experimental" }, @@ -2276,7 +2276,7 @@ "desc": "Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator.", "fieldValue": null, "fieldDefaultValue": null, - "fieldFlag": "ingester.client.initial-connection-window-size", + "fieldFlag": "ingester_client.initial-connection-window-size", "fieldType": "int", "fieldCategory": "experimental" }, @@ -2287,7 +2287,7 @@ "desc": "Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ingester.client.tls-enabled", + "fieldFlag": "ingester_client.tls-enabled", "fieldType": "boolean", "fieldCategory": "advanced" }, @@ -2298,7 +2298,7 @@ "desc": "Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-cert-path", + "fieldFlag": "ingester_client.tls-cert-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2309,7 +2309,7 @@ "desc": "Path to the key for the client certificate. Also requires the client certificate to be configured.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-key-path", + "fieldFlag": "ingester_client.tls-key-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2320,7 +2320,7 @@ "desc": "Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-ca-path", + "fieldFlag": "ingester_client.tls-ca-path", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2331,7 +2331,7 @@ "desc": "Override the expected name on the server certificate.", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-server-name", + "fieldFlag": "ingester_client.tls-server-name", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2342,7 +2342,7 @@ "desc": "Skip validating server certificate.", "fieldValue": null, "fieldDefaultValue": false, - "fieldFlag": "ingester.client.tls-insecure-skip-verify", + "fieldFlag": "ingester_client.tls-insecure-skip-verify", "fieldType": "boolean", "fieldCategory": "advanced" }, @@ -2353,7 +2353,7 @@ "desc": "Override the default cipher suite list (separated by commas). Allowed values:\n\nSecure Ciphers:\n- TLS_AES_128_GCM_SHA256\n- TLS_AES_256_GCM_SHA384\n- TLS_CHACHA20_POLY1305_SHA256\n- TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA\n- TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA\n- TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256\n- TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384\n- TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256\n- TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384\n- TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256\n- TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256\n\nInsecure Ciphers:\n- TLS_RSA_WITH_RC4_128_SHA\n- TLS_RSA_WITH_3DES_EDE_CBC_SHA\n- TLS_RSA_WITH_AES_128_CBC_SHA\n- TLS_RSA_WITH_AES_256_CBC_SHA\n- TLS_RSA_WITH_AES_128_CBC_SHA256\n- TLS_RSA_WITH_AES_128_GCM_SHA256\n- TLS_RSA_WITH_AES_256_GCM_SHA384\n- TLS_ECDHE_ECDSA_WITH_RC4_128_SHA\n- TLS_ECDHE_RSA_WITH_RC4_128_SHA\n- TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA\n- TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256\n- TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256\n", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-cipher-suites", + "fieldFlag": "ingester_client.tls-cipher-suites", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2364,7 +2364,7 @@ "desc": "Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ingester.client.tls-min-version", + "fieldFlag": "ingester_client.tls-min-version", "fieldType": "string", "fieldCategory": "advanced" }, @@ -2375,7 +2375,7 @@ "desc": "The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff.", "fieldValue": null, "fieldDefaultValue": 5000000000, - "fieldFlag": "ingester.client.connect-timeout", + "fieldFlag": "ingester_client.connect-timeout", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -2386,7 +2386,7 @@ "desc": "Initial backoff delay after first connection failure. Only relevant if ConnectTimeout \u003e 0.", "fieldValue": null, "fieldDefaultValue": 1000000000, - "fieldFlag": "ingester.client.connect-backoff-base-delay", + "fieldFlag": "ingester_client.connect-backoff-base-delay", "fieldType": "duration", "fieldCategory": "advanced" }, @@ -2397,9 +2397,20 @@ "desc": "Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout \u003e 0.", "fieldValue": null, "fieldDefaultValue": 5000000000, - "fieldFlag": "ingester.client.connect-backoff-max-delay", + "fieldFlag": "ingester_client.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ingester_client.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -5654,6 +5665,17 @@ "fieldFlag": "querier.frontend-client.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "querier.frontend-client.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -5916,6 +5938,17 @@ "fieldFlag": "querier.scheduler-client.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "querier.scheduler-client.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -6307,6 +6340,17 @@ "fieldFlag": "query-frontend.grpc-client-config.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "query-frontend.grpc-client-config.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -12791,6 +12835,17 @@ "fieldFlag": "ruler.client.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.client.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -13828,6 +13883,17 @@ "fieldFlag": "ruler.query-frontend.grpc-client-config.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "ruler.query-frontend.grpc-client-config.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -16416,6 +16482,17 @@ "fieldFlag": "alertmanager.alertmanager-client.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "alertmanager.alertmanager-client.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -18321,6 +18398,17 @@ "fieldFlag": "query-scheduler.grpc-client-config.connect-backoff-max-delay", "fieldType": "duration", "fieldCategory": "advanced" + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "query-scheduler.grpc-client-config.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, @@ -20293,6 +20381,17 @@ ], "fieldValue": null, "fieldDefaultValue": null + }, + { + "kind": "field", + "name": "cluster_validation_label", + "required": false, + "desc": "Optionally define gRPC client's cluster validation label.", + "fieldValue": null, + "fieldDefaultValue": "", + "fieldFlag": "common.cluster-validation-label", + "fieldType": "string", + "fieldCategory": "experimental" } ], "fieldValue": null, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index ae3fc6f5e65..047298d7119 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -191,6 +191,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -alertmanager.alertmanager-client.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -alertmanager.alertmanager-client.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -alertmanager.alertmanager-client.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -alertmanager.alertmanager-client.connect-backoff-max-delay duration @@ -987,6 +989,8 @@ Usage of ./cmd/mimir/mimir: Maximum number of CPUs that can simultaneously processes WAL replay. If it is set to 0, then each TSDB is replayed with a concurrency equal to the number of CPU cores available on the machine. -blocks-storage.tsdb.wal-segment-size-bytes int TSDB WAL segments files max size (bytes). (default 134217728) + -common.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -common.storage.azure.account-key string Azure storage account key. If unset, Azure managed identities will be used for authentication instead. -common.storage.azure.account-name string @@ -1591,50 +1595,6 @@ Usage of ./cmd/mimir/mimir: After what time a series is considered to be inactive. (default 20m0s) -ingester.active-series-metrics-update-period duration How often to update active series metrics. (default 1m0s) - -ingester.client.backoff-max-period duration - Maximum delay when backing off. (default 10s) - -ingester.client.backoff-min-period duration - Minimum delay when backing off. (default 100ms) - -ingester.client.backoff-on-ratelimits - Enable backoff and retry when we hit rate limits. - -ingester.client.backoff-retries int - Number of times to backoff and retry before failing. (default 10) - -ingester.client.connect-backoff-base-delay duration - Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) - -ingester.client.connect-backoff-max-delay duration - Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0. (default 5s) - -ingester.client.connect-timeout duration - The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff. (default 5s) - -ingester.client.grpc-client-rate-limit float - Rate limit for gRPC client; 0 means disabled. - -ingester.client.grpc-client-rate-limit-burst int - Rate limit burst for gRPC client. - -ingester.client.grpc-compression string - Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2' and '' (disable compression) - -ingester.client.grpc-max-recv-msg-size int - gRPC client max receive message size (bytes). (default 104857600) - -ingester.client.grpc-max-send-msg-size int - gRPC client max send message size (bytes). (default 104857600) - -ingester.client.initial-connection-window-size value - [experimental] Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B) - -ingester.client.initial-stream-window-size value - [experimental] Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B) - -ingester.client.tls-ca-path string - Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used. - -ingester.client.tls-cert-path string - Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured. - -ingester.client.tls-cipher-suites string - Override the default cipher suite list (separated by commas). - -ingester.client.tls-enabled - Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. - -ingester.client.tls-insecure-skip-verify - Skip validating server certificate. - -ingester.client.tls-key-path string - Path to the key for the client certificate. Also requires the client certificate to be configured. - -ingester.client.tls-min-version string - Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13 - -ingester.client.tls-server-name string - Override the expected name on the server certificate. -ingester.error-sample-rate int Each error will be logged once in this many times. Use 0 to log all of them. (default 10) -ingester.ignore-ooo-exemplars @@ -1919,6 +1879,52 @@ Usage of ./cmd/mimir/mimir: [experimental] Period with which to update the per-tenant TSDB configuration. (default 15s) -ingester.use-ingester-owned-series-for-limits [experimental] When enabled, only series currently owned by ingester according to the ring are used when checking user per-tenant series limit. + -ingester_client.backoff-max-period duration + Maximum delay when backing off. (default 10s) + -ingester_client.backoff-min-period duration + Minimum delay when backing off. (default 100ms) + -ingester_client.backoff-on-ratelimits + Enable backoff and retry when we hit rate limits. + -ingester_client.backoff-retries int + Number of times to backoff and retry before failing. (default 10) + -ingester_client.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. + -ingester_client.connect-backoff-base-delay duration + Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) + -ingester_client.connect-backoff-max-delay duration + Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0. (default 5s) + -ingester_client.connect-timeout duration + The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff. (default 5s) + -ingester_client.grpc-client-rate-limit float + Rate limit for gRPC client; 0 means disabled. + -ingester_client.grpc-client-rate-limit-burst int + Rate limit burst for gRPC client. + -ingester_client.grpc-compression string + Use compression when sending messages. Supported values are: 'gzip', 'snappy', 's2' and '' (disable compression) + -ingester_client.grpc-max-recv-msg-size int + gRPC client max receive message size (bytes). (default 104857600) + -ingester_client.grpc-max-send-msg-size int + gRPC client max send message size (bytes). (default 104857600) + -ingester_client.initial-connection-window-size value + [experimental] Initial connection window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B) + -ingester_client.initial-stream-window-size value + [experimental] Initial stream window size. Values less than the default are not supported and are ignored. Setting this to a value other than the default disables the BDP estimator. (default 63KiB1023B) + -ingester_client.tls-ca-path string + Path to the CA certificates to validate server certificate against. If not set, the host's root CA certificates are used. + -ingester_client.tls-cert-path string + Path to the client certificate, which will be used for authenticating with the server. Also requires the key path to be configured. + -ingester_client.tls-cipher-suites string + Override the default cipher suite list (separated by commas). + -ingester_client.tls-enabled + Enable TLS in the gRPC client. This flag needs to be enabled when any other TLS flag is set. If set to false, insecure connection to gRPC server will be used. + -ingester_client.tls-insecure-skip-verify + Skip validating server certificate. + -ingester_client.tls-key-path string + Path to the key for the client certificate. Also requires the client certificate to be configured. + -ingester_client.tls-min-version string + Override the default minimum TLS version. Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13 + -ingester_client.tls-server-name string + Override the expected name on the server certificate. -log.format string Output log messages in the given format. Valid formats: [logfmt, json] (default "logfmt") -log.level value @@ -2115,6 +2121,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -querier.frontend-client.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -querier.frontend-client.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -querier.frontend-client.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -querier.frontend-client.connect-backoff-max-delay duration @@ -2223,6 +2231,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -querier.scheduler-client.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -querier.scheduler-client.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -querier.scheduler-client.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -querier.scheduler-client.connect-backoff-max-delay duration @@ -2305,6 +2315,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -query-frontend.grpc-client-config.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -query-frontend.grpc-client-config.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -query-frontend.grpc-client-config.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -query-frontend.grpc-client-config.connect-backoff-max-delay duration @@ -2521,6 +2533,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -query-scheduler.grpc-client-config.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -query-scheduler.grpc-client-config.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -query-scheduler.grpc-client-config.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -query-scheduler.grpc-client-config.connect-backoff-max-delay duration @@ -2955,6 +2969,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -ruler.client.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -ruler.client.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -ruler.client.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -ruler.client.connect-backoff-max-delay duration @@ -3043,6 +3059,8 @@ Usage of ./cmd/mimir/mimir: Enable backoff and retry when we hit rate limits. -ruler.query-frontend.grpc-client-config.backoff-retries int Number of times to backoff and retry before failing. (default 10) + -ruler.query-frontend.grpc-client-config.cluster-validation-label string + [experimental] Optionally define gRPC client's cluster validation label. -ruler.query-frontend.grpc-client-config.connect-backoff-base-delay duration Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0. (default 1s) -ruler.query-frontend.grpc-client-config.connect-backoff-max-delay duration diff --git a/docs/sources/mimir/configure/configuration-parameters/index.md b/docs/sources/mimir/configure/configuration-parameters/index.md index bc9df10e4d8..d6345a2b8cd 100644 --- a/docs/sources/mimir/configure/configuration-parameters/index.md +++ b/docs/sources/mimir/configure/configuration-parameters/index.md @@ -510,6 +510,10 @@ storage: # system as object storage backend. # The CLI flags prefix for this block configuration is: common.storage [filesystem: ] + +# (experimental) Optionally define gRPC client's cluster validation label. +# CLI flag: -common.cluster-validation-label +[cluster_validation_label: | default = ""] ``` ### server @@ -2669,6 +2673,10 @@ alertmanager_client: # CLI flag: -alertmanager.alertmanager-client.connect-backoff-max-delay [connect_backoff_max_delay: | default = 5s] + # (experimental) Optionally define gRPC client's cluster validation label. + # CLI flag: -alertmanager.alertmanager-client.cluster-validation-label + [cluster_validation_label: | default = ""] + # (advanced) The interval between persisting the current alertmanager state # (notification log and silences) to object storage. This is only used when # sharding is enabled. This state is read when all replicas for a shard can not @@ -2776,7 +2784,7 @@ The `ingester_client` block configures how the distributors connect to the inges ```yaml # Configures the gRPC client used to communicate with ingesters from # distributors, queriers and rulers. -# The CLI flags prefix for this block configuration is: ingester.client +# The CLI flags prefix for this block configuration is: ingester_client [grpc_client_config: ] ``` @@ -2784,7 +2792,7 @@ The `ingester_client` block configures how the distributors connect to the inges The `grpc_client` block configures the gRPC client used to communicate between two Mimir components. The supported CLI flags `` used to reference this configuration block are: -- `ingester.client` +- `ingester_client` - `querier.frontend-client` - `querier.scheduler-client` - `query-frontend.grpc-client-config` @@ -2927,6 +2935,10 @@ backoff_config: # if ConnectTimeout > 0. # CLI flag: -.connect-backoff-max-delay [connect_backoff_max_delay: | default = 5s] + +# (experimental) Optionally define gRPC client's cluster validation label. +# CLI flag: -.cluster-validation-label +[cluster_validation_label: | default = ""] ``` ### frontend_worker diff --git a/go.mod b/go.mod index d53ba8e6036..6cf6bc2c9ee 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/gopacket v1.1.19 github.com/gorilla/mux v1.8.1 - github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237 + github.com/grafana/dskit v0.0.0-20250303214858-d23654211757 github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/influxdata/influxdb/v2 v2.7.11 diff --git a/go.sum b/go.sum index ef764b1d60e..8035681522b 100644 --- a/go.sum +++ b/go.sum @@ -1272,8 +1272,8 @@ github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc h1:PXZQA2WCxe85T github.com/grafana-tools/sdk v0.0.0-20220919052116-6562121319fc/go.mod h1:AHHlOEv1+GGQ3ktHMlhuTUwo3zljV3QJbC0+8o2kn+4= github.com/grafana/alerting v0.0.0-20250225150117-15e285d78df2 h1:kESrzm0FcRVLmGIQCgl1MCwDGLH4sLzWphr7mcFdbfI= github.com/grafana/alerting v0.0.0-20250225150117-15e285d78df2/go.mod h1:hdGB3dSl8Ma9Rjo2YiAEAjMkZ5HiNJbNDqRKDefRZrM= -github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237 h1:VZagYtPcmjgazfPAuWN7lER6mprG20r51+1eYPpATkw= -github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237/go.mod h1:cu2zIOHhAgRaIDuECsERftSp1l7KHq1aX1jgihQCu0c= +github.com/grafana/dskit v0.0.0-20250303214858-d23654211757 h1:nAd6h3RfteaAMeTO4cJLcPQGm1X7uYxv5oAhZICkBNw= +github.com/grafana/dskit v0.0.0-20250303214858-d23654211757/go.mod h1:cu2zIOHhAgRaIDuECsERftSp1l7KHq1aX1jgihQCu0c= github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc h1:BW+LjKJDz0So5LI8UZfW5neWeKpSkWqhmGjQFzcFfLM= github.com/grafana/e2e v0.1.2-0.20240118170847-db90b84177fc/go.mod h1:JVmqPBe8A/pZWwRoJW5ZjyALeY5OXMzPl7LrVXOdZAI= github.com/grafana/franz-go v0.0.0-20241009100846-782ba1442937 h1:fwwnG/NcygoS6XbAaEyK2QzMXI/BZIEJvQ3CD+7XZm8= diff --git a/integration/ingester_test.go b/integration/ingester_test.go index adb5574c462..8c3bf9480ed 100644 --- a/integration/ingester_test.go +++ b/integration/ingester_test.go @@ -849,6 +849,7 @@ func TestInvalidClusterValidationLabel(t *testing.T) { baseFlags := map[string]string{ "-distributor.ingestion-tenant-shard-size": "0", "-ingester.ring.heartbeat-period": "1s", + "-common.cluster-validation-label": testCase.distributorClusterLabel, } flags := mergeFlags( @@ -857,13 +858,6 @@ func TestInvalidClusterValidationLabel(t *testing.T) { baseFlags, ) - distributorFlags := mergeFlags( - flags, - map[string]string{ - "-server.cluster-validation.label": testCase.distributorClusterLabel, - }, - ) - ingesterFlags := mergeFlags( flags, map[string]string{ @@ -879,7 +873,7 @@ func TestInvalidClusterValidationLabel(t *testing.T) { require.NoError(t, s.StartAndWaitReady(consul, minio)) // Start Mimir components. - distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), distributorFlags) + distributor := e2emimir.NewDistributor("distributor", consul.NetworkHTTPEndpoint(), flags) ingester := e2emimir.NewIngester("ingester", consul.NetworkHTTPEndpoint(), ingesterFlags) require.NoError(t, s.StartAndWaitReady(distributor, ingester)) diff --git a/pkg/ingester/client/client.go b/pkg/ingester/client/client.go index be8a7669511..f561a8b7e4a 100644 --- a/pkg/ingester/client/client.go +++ b/pkg/ingester/client/client.go @@ -38,8 +38,8 @@ func MakeIngesterClient(inst ring.InstanceDesc, cfg Config, metrics *Metrics, lo reportGRPCStatusesOptions := []middleware.InstrumentationOption{middleware.ReportGRPCStatusOption} unary, stream := grpcclient.Instrument(metrics.requestDuration, reportGRPCStatusesOptions...) unary = append(unary, querierapi.ReadConsistencyClientUnaryInterceptor) - if cfg.ClusterValidationLabel != "" { - unary = append(unary, middleware.ClusterUnaryClientInterceptor(cfg.ClusterValidationLabel, metrics.invalidClusterVerificationLabels, logger)) + if cfg.GRPCClientConfig.ClusterValidationLabel != "" { + unary = append(unary, middleware.ClusterUnaryClientInterceptor(cfg.GRPCClientConfig.ClusterValidationLabel, metrics.invalidClusterVerificationLabels, logger)) } stream = append(stream, querierapi.ReadConsistencyClientStreamInterceptor) @@ -70,8 +70,7 @@ func (c *closableHealthAndIngesterClient) Close() error { // Config is the configuration struct for the ingester client type Config struct { - GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with ingesters from distributors, queriers and rulers."` - ClusterValidationLabel string `yaml:"-"` + GRPCClientConfig grpcclient.Config `yaml:"grpc_client_config" doc:"description=Configures the gRPC client used to communicate with ingesters from distributors, queriers and rulers."` } // RegisterFlags registers configuration settings used by the ingester client config. diff --git a/pkg/mimir/mimir.go b/pkg/mimir/mimir.go index d766e59545f..f27f8473199 100644 --- a/pkg/mimir/mimir.go +++ b/pkg/mimir/mimir.go @@ -21,6 +21,7 @@ import ( "github.com/go-kit/log/level" "github.com/gorilla/mux" "github.com/grafana/dskit/flagext" + "github.com/grafana/dskit/grpcclient" "github.com/grafana/dskit/grpcutil" "github.com/grafana/dskit/kv/memberlist" "github.com/grafana/dskit/modules" @@ -223,6 +224,9 @@ func (c *Config) CommonConfigInheritance() CommonConfigInheritance { "ruler_storage": &c.RulerStorage.StorageBackendConfig, "alertmanager_storage": &c.AlertmanagerStorage.StorageBackendConfig, }, + ClusterValidationLabel: map[string]*grpcclient.Config{ + "ingester_client": &c.IngesterClient.GRPCClientConfig, + }, } } @@ -626,6 +630,29 @@ func InheritCommonFlagValues(log log.Logger, fs *flag.FlagSet, common CommonConf return fmt.Errorf("can't inherit common flags for %q: %w", desc, err) } } + if common.ClusterValidationLabel == "" { + // Nothing to inherit because origin was not set. + continue + } + for _, grpcClientConfig := range inheritance.ClusterValidationLabel { + if grpcClientConfig == nil { + continue + } + if grpcClientConfig.ClusterValidationLabel != "" { + // Can't inherit because destination was set. + continue + } + if common.ClusterValidationLabel == grpcClientConfig.ClusterValidationLabel { + // Already the same, no need to touch. + continue + } + level.Debug(log).Log( + "msg", "Inheriting flag value", + "flag_name", "cluster_value_label", "origin_value", common.ClusterValidationLabel, + "destination_value", grpcClientConfig.ClusterValidationLabel, + ) + grpcClientConfig.ClusterValidationLabel = common.ClusterValidationLabel + } } return nil @@ -663,16 +690,19 @@ func inheritFlags(log log.Logger, orig util.RegisteredFlags, dest util.Registere } type CommonConfig struct { - Storage bucket.StorageBackendConfig `yaml:"storage"` + Storage bucket.StorageBackendConfig `yaml:"storage"` + ClusterValidationLabel string `yaml:"cluster_validation_label" category:"experimental"` } type CommonConfigInheritance struct { - Storage map[string]*bucket.StorageBackendConfig + Storage map[string]*bucket.StorageBackendConfig + ClusterValidationLabel map[string]*grpcclient.Config } // RegisterFlags registers flag. func (c *CommonConfig) RegisterFlags(f *flag.FlagSet) { c.Storage.RegisterFlagsWithPrefix("common.storage.", f) + f.StringVar(&c.ClusterValidationLabel, "common.cluster-validation-label", "", "Optionally define gRPC client's cluster validation label.") } // configWithCustomCommonUnmarshaler unmarshals config with custom unmarshaler for the `common` field. diff --git a/pkg/mimir/mimir_config_test.go b/pkg/mimir/mimir_config_test.go index 0d36ece941d..2ad90cb816e 100644 --- a/pkg/mimir/mimir_config_test.go +++ b/pkg/mimir/mimir_config_test.go @@ -20,13 +20,14 @@ func TestCommonConfigCanBeExtended(t *testing.T) { fs := flag.NewFlagSet("test", flag.PanicOnError) cfg.RegisterFlags(fs, log.NewNopLogger()) - args := []string{"-common.storage.backend", "s3"} + args := []string{"-common.storage.backend", "s3", "-common.cluster-validation-label", "cluster"} require.NoError(t, fs.Parse(args)) require.NoError(t, mimir.InheritCommonFlagValues(log.NewNopLogger(), fs, cfg.MimirConfig.Common, &cfg.MimirConfig, &cfg)) - // Value should be properly inherited. + // Values should be properly inherited. require.Equal(t, "s3", cfg.CustomStorage.Backend) + require.Equal(t, "cluster", cfg.MimirConfig.IngesterClient.GRPCClientConfig.ClusterValidationLabel) // Mimir's inheritance should still work. require.Equal(t, "s3", cfg.MimirConfig.BlocksStorage.Bucket.Backend) @@ -37,6 +38,7 @@ func TestCommonConfigCanBeExtended(t *testing.T) { common: storage: backend: s3 + cluster_validation_label: cluster ` var cfg customExtendedConfig @@ -46,8 +48,9 @@ common: err := yaml.Unmarshal([]byte(commonYAMLConfig), &cfg) require.NoError(t, err) - // Value should be properly inherited. + // Values should be properly inherited. require.Equal(t, "s3", cfg.CustomStorage.Backend) + require.Equal(t, "cluster", cfg.MimirConfig.IngesterClient.GRPCClientConfig.ClusterValidationLabel) // Mimir's inheritance should still work. require.Equal(t, "s3", cfg.MimirConfig.BlocksStorage.Bucket.Backend) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 0459d10c5ca..cea5f08b186 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -472,7 +472,6 @@ func (t *Mimir) initDistributorService() (serv services.Service, err error) { t.Cfg.Distributor.MinimiseIngesterRequestsHedgingDelay = t.Cfg.Querier.MinimiseIngesterRequestsHedgingDelay t.Cfg.Distributor.PreferAvailabilityZone = t.Cfg.Querier.PreferAvailabilityZone t.Cfg.Distributor.IngestStorageConfig = t.Cfg.IngestStorage - t.Cfg.IngesterClient.ClusterValidationLabel = t.Cfg.Server.ClusterValidation.Label t.Distributor, err = distributor.New(t.Cfg.Distributor, t.Cfg.IngesterClient, t.Overrides, t.ActiveGroupsCleanup, t.CostAttributionManager, t.IngesterRing, t.IngesterPartitionInstanceRing, diff --git a/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go b/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go index a8f728c61e2..7655ec671b0 100644 --- a/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go +++ b/vendor/github.com/grafana/dskit/grpcclient/grpcclient.go @@ -45,6 +45,8 @@ type Config struct { // CustomCompressors allows configuring custom compressors. CustomCompressors []string `yaml:"-"` + + ClusterValidationLabel string `yaml:"cluster_validation_label" category:"experimental"` } // RegisterFlags registers flags. @@ -82,6 +84,7 @@ func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { f.DurationVar(&cfg.ConnectTimeout, prefix+".connect-timeout", 5*time.Second, "The maximum amount of time to establish a connection. A value of 0 means default gRPC client connect timeout and backoff.") f.DurationVar(&cfg.ConnectBackoffBaseDelay, prefix+".connect-backoff-base-delay", time.Second, "Initial backoff delay after first connection failure. Only relevant if ConnectTimeout > 0.") f.DurationVar(&cfg.ConnectBackoffMaxDelay, prefix+".connect-backoff-max-delay", 5*time.Second, "Maximum backoff delay when establishing a connection. Only relevant if ConnectTimeout > 0.") + f.StringVar(&cfg.ClusterValidationLabel, prefix+".cluster-validation-label", "", "Optionally define gRPC client's cluster validation label.") cfg.BackoffConfig.RegisterFlagsWithPrefix(prefix, f) diff --git a/vendor/modules.txt b/vendor/modules.txt index 33a47da242d..f2df04a6de6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -651,7 +651,7 @@ github.com/grafana/alerting/receivers/webex github.com/grafana/alerting/receivers/webhook github.com/grafana/alerting/receivers/wecom github.com/grafana/alerting/templates -# github.com/grafana/dskit v0.0.0-20250303172748-fd4441b85237 +# github.com/grafana/dskit v0.0.0-20250303214858-d23654211757 ## explicit; go 1.21 github.com/grafana/dskit/backoff github.com/grafana/dskit/ballast