frontend: Add integration test for split and cache middleware (#10798)

* frontend: Add integration test for split and cache middleware
This tests two aspects of the split and cache feature:
1. We can go above the partial query length (because queries are split)
2. The query is split into multiple 24h queries

This is in response to #10796

* Update integration/query_frontend_test.go

Co-authored-by: Dimitar Dimitrov <[email protected]>

---------

Co-authored-by: Dimitar Dimitrov <[email protected]>
(cherry picked from commit b561724)

integration/query_frontend_test.go

@@ -270,6 +270,7 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 	configFile, flags := cfg.setup(t, s)
 
 	flags = mergeFlags(flags, map[string]string{
+		"-querier.max-partial-query-length":                      "30d",
 		"-query-frontend.cache-results":                          "true",
 		"-query-frontend.results-cache.backend":                  "memcached",
 		"-query-frontend.results-cache.memcached.addresses":      "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort),
@@ -364,6 +365,16 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 		c, err := e2emimir.NewClient("", queryFrontend.HTTPEndpoint(), "", "", fmt.Sprintf("user-%d", userID))
 		require.NoError(t, err)
 
+		// Do a long query to test the split and cache middleware.
+		if userID == 0 {
+			require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(0), "cortex_frontend_split_queries_total"))
+			end := time.Now()
+			start := end.Add(-(30*24*time.Hour + 1*time.Hour)) // 30 days + 1 hour. Makes sure we can go above the max partial query length.
+			_, err = c.QueryRange("{instance=~\"hello.*\"}", start, end, time.Hour)
+			require.NoError(t, err)
+			require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(31), "cortex_frontend_split_queries_total"))
+		}
+
 		// No need to repeat the test on start/end time rounding for each user.
 		if userID == 0 {
 			start := time.Unix(1595846748, 806*1e6)
@@ -397,6 +408,7 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 			require.Len(t, result, 0)
 		}
 
+		// Test subquery spin-off.
 		if userID == 0 {
 			require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(0), "cortex_frontend_spun_off_subqueries_total"))
 			result, err := c.Query("sum_over_time(((count(series_1) * count(series_1)) or vector(1))[6h:15m])", now)
@@ -422,10 +434,11 @@ func runQueryFrontendTest(t *testing.T, cfg queryFrontendTestConfig) {
 	wg.Wait()
 
 	// Compute the expected number of queries.
-	expectedQueriesCount := float64(numUsers*numQueriesPerUser) + 3
+	expectedQueriesCount := float64(numUsers*numQueriesPerUser) + 4
 	// The "time()" query and the query with time range < "query ingesters within" are not pushed down to ingesters.
+	// +1 because one split query ends up touching the ingester.
 	// +2 because the spun off subquery ends up as additional ingester queries.
-	expectedIngesterQueriesCount := float64(numUsers*numQueriesPerUser) + 2
+	expectedIngesterQueriesCount := float64(numUsers*numQueriesPerUser) + 3
 	if cfg.queryStatsEnabled {
 		expectedQueriesCount++
 		expectedIngesterQueriesCount++