Rename metric to cortex_querier_query_storegateway_chunks_total (#7145)

- Rename cortex_query_storegateway_chunks_total to cortex_querier_query_storegateway_chunks_total
- Add metric to test cases

CHANGELOG.md

@@ -35,7 +35,7 @@
 * [ENHANCEMENT] Distributor: support disabling metric relabel rules per-tenant via the flag `-distributor.metric-relabeling-enabled` or associated YAML. #6970
 * [ENHANCEMENT] Distributor: `-distributor.remote-timeout` is now accounted from the first ingester push request being sent. #6972
 * [ENHANCEMENT] Storage Provider: allow aws sts support for s3 storage provider #6172
-* [ENHANCEMENT] Querier: add `cortex_querier_queries_storage_type_total ` metric that indicates how many queries have executed for a source, ingesters or store-gateways. Add `cortex_query_storegateway_chunks_total` metric to count the number of chunks fetched from a store gateway. #7099
+* [ENHANCEMENT] Querier: add `cortex_querier_queries_storage_type_total ` metric that indicates how many queries have executed for a source, ingesters or store-gateways. Add `cortex_querier_query_storegateway_chunks_total` metric to count the number of chunks fetched from a store gateway. #7099,#7145
 * [ENHANCEMENT] Query-frontend: add experimental support for sharding active series queries via `-query-frontend.shard-active-series-queries`. #6784
 * [ENHANCEMENT] Distributor: set `-distributor.reusable-ingester-push-workers=2000` by default and mark feature as `advanced`. #7128
 * [ENHANCEMENT] All: set `-server.grpc.num-workers=100` by default and mark feature as `advanced`. #7131

integration/asserts.go

@@ -33,8 +33,8 @@ var (
 	// Service-specific metrics prefixes which shouldn't be used by any other service.
 	serviceMetricsPrefixes = map[ServiceType][]string{
 		Distributor:    {},
-		Ingester:       {"!cortex_ingester_client", "cortex_ingester"},                                                          // The metrics prefix cortex_ingester_client may be used by other components so we ignore it.
-		Querier:        {"!cortex_querier_storegateway", "!cortex_querier_blocks", "!cortex_querier_queries", "cortex_querier"}, // The metrics prefix cortex_querier_storegateway, cortex_querier_blocks and cortex_querier_queries may be used by other components so we ignore it.
+		Ingester:       {"!cortex_ingester_client", "cortex_ingester"},                                                                                   // The metrics prefix cortex_ingester_client may be used by other components so we ignore it.
+		Querier:        {"!cortex_querier_storegateway", "!cortex_querier_blocks", "!cortex_querier_queries", "!cortex_querier_query", "cortex_querier"}, // The metrics prefix cortex_querier_storegateway, cortex_querier_blocks, cortex_querier_queries, and cortex_querier_query may be used by other components so we ignore it.
 		QueryFrontend:  {"cortex_frontend", "cortex_query_frontend"},
 		QueryScheduler: {"cortex_query_scheduler"},
 		AlertManager:   {"cortex_alertmanager"},

pkg/querier/blocks_store_queryable.go

@@ -137,9 +137,8 @@ func newBlocksStoreQueryableMetrics(reg prometheus.Registerer) *blocksStoreQuery
 			Name: "cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total",
 			Help: "Blocks that couldn't be checked for query and compactor sharding optimization due to incompatible shard counts.",
 		}),
-		// Named to be consistent with distributor_query_ingester_chunks_total
 		chunksTotal: promauto.With(reg).NewCounter(prometheus.CounterOpts{
-			Name: "cortex_query_storegateway_chunks_total",
+			Name: "cortex_querier_query_storegateway_chunks_total",
 			Help: "Number of chunks received from store gateways at query time.",
 		}),
 	}

pkg/querier/blocks_store_queryable_test.go

@@ -271,7 +271,7 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 			storeSetResponses: []interface{}{
 				map[BlocksStoreClient][]ulid.ULID{
 					&storeGatewayClientMock{remoteAddr: "1.1.1.1", mockedSeriesResponses: []*storepb.SeriesResponse{
-						mockSeriesResponse(series1Label, minT+1, 2),
+						mockSeriesResponse(series1Label, minT+1, 2), // a chunk is written for each mockSeriesResponse
 						mockSeriesResponse(series2Label, minT, 1),
 						mockHintsResponse(block1),
 					}}: {block1},
@@ -340,6 +340,9 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 				# HELP cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total Blocks that couldn't be checked for query and compactor sharding optimization due to incompatible shard counts.
 				# TYPE cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total counter
 				cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total 0
+				# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+				# TYPE cortex_querier_query_storegateway_chunks_total counter
+				cortex_querier_query_storegateway_chunks_total 6
 			`,
 		},
 		"a single store-gateway instance has some missing blocks (consistency check failed)": {
@@ -476,6 +479,9 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 				# HELP cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total Blocks that couldn't be checked for query and compactor sharding optimization due to incompatible shard counts.
 				# TYPE cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total counter
 				cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total 0
+				# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+				# TYPE cortex_querier_query_storegateway_chunks_total counter
+				cortex_querier_query_storegateway_chunks_total 4
 			`,
 		},
 		"max chunks per query limit greater then the number of chunks fetched": {
@@ -679,6 +685,9 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 					cortex_querier_storegateway_refetches_per_query_bucket{le="+Inf"} 1
 					cortex_querier_storegateway_refetches_per_query_sum 0
 					cortex_querier_storegateway_refetches_per_query_count 1
+					# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+					# TYPE cortex_querier_query_storegateway_chunks_total counter
+					cortex_querier_query_storegateway_chunks_total 2
 			`,
 		},
 		"all blocks are queried if shards don't match": {
@@ -746,6 +755,9 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 					cortex_querier_storegateway_refetches_per_query_bucket{le="+Inf"} 1
 					cortex_querier_storegateway_refetches_per_query_sum 0
 					cortex_querier_storegateway_refetches_per_query_count 1
+					# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+					# TYPE cortex_querier_query_storegateway_chunks_total counter
+					cortex_querier_query_storegateway_chunks_total 2
 			`,
 		},
 		"multiple store-gateways have the block, but one of them fails to return": {
@@ -813,6 +825,9 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 					cortex_querier_storegateway_refetches_per_query_bucket{le="+Inf"} 1
 					cortex_querier_storegateway_refetches_per_query_sum 1
 					cortex_querier_storegateway_refetches_per_query_count 1
+					# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+					# TYPE cortex_querier_query_storegateway_chunks_total counter
+					cortex_querier_query_storegateway_chunks_total 1
 			`,
 		},
 	}
@@ -942,7 +957,8 @@ func TestBlocksStoreQuerier_Select(t *testing.T) {
 					if testData.expectedMetrics != "" {
 						assert.NoError(t, testutil.GatherAndCompare(reg, strings.NewReader(testData.expectedMetrics),
 							"cortex_querier_storegateway_instances_hit_per_query", "cortex_querier_storegateway_refetches_per_query",
-							"cortex_querier_blocks_found_total", "cortex_querier_blocks_queried_total", "cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total"))
+							"cortex_querier_blocks_found_total", "cortex_querier_blocks_queried_total", "cortex_querier_blocks_with_compactor_shard_but_incompatible_query_shard_total",
+							"cortex_querier_query_storegateway_chunks_total"))
 					}
 				})
 			}
@@ -1618,6 +1634,9 @@ func TestBlocksStoreQuerier_Labels(t *testing.T) {
 				cortex_querier_storegateway_refetches_per_query_bucket{le="+Inf"} 1
 				cortex_querier_storegateway_refetches_per_query_sum 2
 				cortex_querier_storegateway_refetches_per_query_count 1
+				# HELP cortex_querier_query_storegateway_chunks_total Number of chunks received from store gateways at query time.
+				# TYPE cortex_querier_query_storegateway_chunks_total counter
+				cortex_querier_query_storegateway_chunks_total 4
 			`,
 		},
 		"multiple store-gateways have the block, but one of them fails to return": {