feat: adding consumer-group policy overrides support in deck (#1518)

* feat: ading consumer-group policy overrides support in deck * chore: updated gdr version to v1.20.0 * tests: added integration tests for sync and dump with --consumer-group-policy-overrides * fix: fixed tag description * tests: separated Konnect tests to ensure kong version retrieval does not fail (#1519) * tests: separated Konnect tests to ensure kong version retrieval does not fail * tests: fixed test for dql routes in konnect * tests: fixed test condition * tests: fixed deck token setting position
Kong · Feb 7, 2025 · 5f266f9 · 5f266f9
1 parent 107c160
commit 5f266f9
Show file tree

Hide file tree

Showing 16 changed files with 1,731 additions and 131 deletions.
diff --git a/cmd/common.go b/cmd/common.go
@@ -257,6 +257,8 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 		}
 	}
 
+	dumpConfig.IsConsumerGroupPolicyOverrideSet = determinePolicyOverride(*targetContent, dumpConfig)
+
 	dumpConfig.SelectorTags, err = determineSelectorTag(*targetContent, dumpConfig)
 	if err != nil {
 		return err
@@ -420,6 +422,18 @@ func syncMain(ctx context.Context, filenames []string, dry bool, parallelism,
 	return nil
 }
 
+func determinePolicyOverride(targetContent file.Content, config dump.Config) bool {
+	if config.IsConsumerGroupPolicyOverrideSet {
+		return true
+	}
+
+	if targetContent.Info != nil && targetContent.Info.ConsumerGroupPolicyOverrides {
+		return targetContent.Info.ConsumerGroupPolicyOverrides
+	}
+
+	return false
+}
+
 func determineLookUpSelectorTagsConsumerGroups(targetContent file.Content) ([]string, error) {
 	if targetContent.Info != nil &&
 		targetContent.Info.LookUpSelectorTags != nil &&

diff --git a/cmd/gateway_diff.go b/cmd/gateway_diff.go
@@ -101,6 +101,10 @@ that will be created, updated, or deleted.
 		false, "do not diff CA certificates.")
 	diffCmd.Flags().BoolVar(&diffJSONOutput, "json-output",
 		false, "generate command execution report in a JSON format")
+	diffCmd.Flags().BoolVar(&dumpConfig.IsConsumerGroupPolicyOverrideSet, "consumer-group-policy-overrides",
+		false, "allow deck to diff consumer-group policy overrides.\n"+
+			"This allows policy overrides to work with Kong GW versions >= 3.4\n"+
+			"Warning: do not mix with consumer-group scoped plugins")
 	addSilenceEventsFlag(diffCmd.Flags())
 	return diffCmd
 }
diff --git a/cmd/gateway_dump.go b/cmd/gateway_dump.go
@@ -93,12 +93,13 @@ func executeDump(cmd *cobra.Command, _ []string) error {
 			}
 
 			if err := file.KongStateToFile(ks, file.WriteConfig{
-				SelectTags:  dumpConfig.SelectorTags,
-				Workspace:   workspace,
-				Filename:    workspace,
-				FileFormat:  format,
-				WithID:      dumpWithID,
-				KongVersion: kongVersion,
+				SelectTags:                       dumpConfig.SelectorTags,
+				Workspace:                        workspace,
+				Filename:                         workspace,
+				FileFormat:                       format,
+				WithID:                           dumpWithID,
+				KongVersion:                      kongVersion,
+				IsConsumerGroupPolicyOverrideSet: dumpConfig.IsConsumerGroupPolicyOverrideSet,
 			}); err != nil {
 				return err
 			}
@@ -132,12 +133,13 @@ func executeDump(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("building state: %w", err)
 	}
 	return file.KongStateToFile(ks, file.WriteConfig{
-		SelectTags:  dumpConfig.SelectorTags,
-		Workspace:   dumpWorkspace,
-		Filename:    dumpCmdKongStateFile,
-		FileFormat:  format,
-		WithID:      dumpWithID,
-		KongVersion: kongVersion,
+		SelectTags:                       dumpConfig.SelectorTags,
+		Workspace:                        dumpWorkspace,
+		Filename:                         dumpCmdKongStateFile,
+		FileFormat:                       format,
+		WithID:                           dumpWithID,
+		KongVersion:                      kongVersion,
+		IsConsumerGroupPolicyOverrideSet: dumpConfig.IsConsumerGroupPolicyOverrideSet,
 	})
 }
 
@@ -197,6 +199,10 @@ configure Kong.`,
 		false, "do not show the association between consumer and consumer-group.\n"+
 			"If set to true, deck skips listing consumers with consumer-groups,\n"+
 			"thus gaining some performance with large configs. This flag is not valid with Konnect.")
+	dumpCmd.Flags().BoolVar(&dumpConfig.IsConsumerGroupPolicyOverrideSet, "consumer-group-policy-overrides",
+		false, "allow deck to dump consumer-group policy overrides.\n"+
+			"This allows policy overrides to work with Kong GW versions >= 3.4\n"+
+			"Warning: do not mix with consumer-group scoped plugins")
 	if deprecated {
 		dumpCmd.Flags().StringVarP(&dumpCmdKongStateFileDeprecated, "output-file", "o",
 			fileOutDefault, "file to which to write Kong's configuration."+

diff --git a/cmd/gateway_sync.go b/cmd/gateway_sync.go
@@ -96,6 +96,10 @@ to get Kong's state in sync with the input state.`,
 			"See `db_update_propagation` in kong.conf.")
 	syncCmd.Flags().BoolVar(&dumpConfig.SkipCACerts, "skip-ca-certificates",
 		false, "do not sync CA certificates.")
+	syncCmd.Flags().BoolVar(&dumpConfig.IsConsumerGroupPolicyOverrideSet, "consumer-group-policy-overrides",
+		false, "allow deck to sync consumer-group policy overrides.\n"+
+			"This allows policy overrides to work with Kong GW versions >= 3.4\n"+
+			"Warning: do not mix with consumer-group scoped plugins")
 	syncCmd.Flags().BoolVar(&syncJSONOutput, "json-output",
 		false, "generate command execution report in a JSON format")
 	addSilenceEventsFlag(syncCmd.Flags())

diff --git a/go.mod b/go.mod
@@ -13,7 +13,7 @@ require (
 	github.com/fatih/color v1.17.0
 	github.com/google/go-cmp v0.6.0
 	github.com/kong/go-apiops v0.1.41
-	github.com/kong/go-database-reconciler v1.19.3
+	github.com/kong/go-database-reconciler v1.20.1
 	github.com/kong/go-kong v0.63.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/spf13/cobra v1.8.1

diff --git a/go.sum b/go.sum
@@ -247,8 +247,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q
 github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kong/go-apiops v0.1.41 h1:1KXbQqyhO2E4nEnXJNqUDmHZU/LQ1U7Zoi+MAlcM2P0=
 github.com/kong/go-apiops v0.1.41/go.mod h1:sATq9Tz+ivzHKZU+tDXkRtEZnf64xroU3lgv3yXqP4I=
-github.com/kong/go-database-reconciler v1.19.3 h1:wcvRIL1lt21ZDCGH0OaGbyTWxKv8tiGpbSCP4hUp6Ew=
-github.com/kong/go-database-reconciler v1.19.3/go.mod h1:crxjFGiSSKWtP1AqFZz/IEnlxTX/hFkcBdMdwQbEpmk=
+github.com/kong/go-database-reconciler v1.20.1 h1:mXS4sThlsZAJda+jX3Gf2JCdqQZx7FWSuhMIuwVjKg0=
+github.com/kong/go-database-reconciler v1.20.1/go.mod h1:crxjFGiSSKWtP1AqFZz/IEnlxTX/hFkcBdMdwQbEpmk=
 github.com/kong/go-kong v0.63.0 h1:8ECLgkgDqON61qCMq/M0gTwZKYxg55Oy692dRDOOBiU=
 github.com/kong/go-kong v0.63.0/go.mod h1:ma9GWnhkxtrXZlLFfED955HjVzmUojYEHet3lm+PDik=
 github.com/kong/go-slugify v1.0.0 h1:vCFAyf2sdoSlBtLcrmDWUFn0ohlpKiKvQfXZkO5vSKY=

diff --git a/tests/integration/dump_test.go b/tests/integration/dump_test.go
@@ -450,3 +450,66 @@ func Test_SkipConsumersWithConsumerGroups(t *testing.T) {
 		})
 	}
 }
+
+func Test_Dump_ConsumerGroupPlugin_PolicyOverrides(t *testing.T) {
+	tests := []struct {
+		name          string
+		stateFile     string
+		expectedFile  string
+		errorExpected bool
+		errorString   string
+		runWhen       func(t *testing.T)
+	}{
+		{
+			name:          "dump with flag --consumer-group-policy-overrides set: >=3.4.0 <3.8.0",
+			stateFile:     "testdata/sync/037-consumer-group-policy-overrides/kong34x-no-info.yaml",
+			expectedFile:  "testdata/sync/037-consumer-group-policy-overrides/kong34x.yaml",
+			errorExpected: false,
+			runWhen:       func(t *testing.T) { runWhen(t, "enterprise", ">=3.4.0 <3.8.0") },
+		},
+		{
+			name:          "dump with flag --consumer-group-policy-overrides set: >=3.8.0 <3.9.0",
+			stateFile:     "testdata/sync/037-consumer-group-policy-overrides/kong38x-no-info.yaml",
+			expectedFile:  "testdata/sync/037-consumer-group-policy-overrides/kong38x.yaml",
+			errorExpected: false,
+			runWhen:       func(t *testing.T) { runWhen(t, "enterprise", ">=3.8.0 <3.9.0") },
+		},
+		{
+			name:          "dump with flag --consumer-group-policy-overrides set: >=3.9.0",
+			stateFile:     "testdata/sync/037-consumer-group-policy-overrides/kong39x-no-info.yaml",
+			expectedFile:  "testdata/sync/037-consumer-group-policy-overrides/kong39x.yaml",
+			errorExpected: false,
+			runWhen:       func(t *testing.T) { runWhen(t, "enterprise", ">=3.9.0") },
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			tc.runWhen(t)
+			setup(t)
+
+			require.NoError(t, sync(context.Background(), tc.stateFile, "--consumer-group-policy-overrides"))
+
+			var (
+				output string
+				err    error
+			)
+
+			output, err = dump(
+				"--consumer-group-policy-overrides",
+				"-o", "-",
+			)
+
+			if tc.errorExpected {
+				assert.Equal(t, err.Error(), tc.errorString)
+				return
+			}
+
+			require.NoError(t, err)
+
+			expected, err := readFile(tc.expectedFile)
+			require.NoError(t, err)
+			assert.Equal(t, expected, output)
+		})
+	}
+}
diff --git a/tests/integration/reset_test.go b/tests/integration/reset_test.go
@@ -66,7 +66,7 @@ func Test_Reset_SkipCACert_2x(t *testing.T) {
 
 			require.NoError(t, sync(context.Background(), tc.kongFile))
 			reset(t, "--skip-ca-certificates")
-			testKongState(t, client, false, tc.expectedState, nil)
+			testKongState(t, client, false, false, tc.expectedState, nil)
 		})
 	}
 }
@@ -100,19 +100,31 @@ func Test_Reset_SkipCACert_3x(t *testing.T) {
 
 			require.NoError(t, sync(context.Background(), tc.kongFile))
 			reset(t, "--skip-ca-certificates")
-			testKongState(t, client, false, tc.expectedState, nil)
+			testKongState(t, client, false, false, tc.expectedState, nil)
 		})
 	}
 }
 
 func Test_Reset_ConsumerGroupConsumersWithCustomID(t *testing.T) {
-	runWhenEnterpriseOrKonnect(t, ">=3.0.0")
+	runWhen(t, "enterprise", ">=3.0.0")
 	setup(t)
 
 	client, err := getTestClient()
 	require.NoError(t, err)
 
 	require.NoError(t, sync(context.Background(), "testdata/sync/028-consumer-group-consumers-custom_id/kong.yaml"))
 	reset(t)
-	testKongState(t, client, false, utils.KongRawState{}, nil)
+	testKongState(t, client, false, false, utils.KongRawState{}, nil)
+}
+
+func Test_Reset_ConsumerGroupConsumersWithCustomID_Konnect(t *testing.T) {
+	runWhenKonnect(t)
+	setup(t)
+
+	client, err := getTestClient()
+	require.NoError(t, err)
+
+	require.NoError(t, sync(context.Background(), "testdata/sync/028-consumer-group-consumers-custom_id/kong.yaml"))
+	reset(t)
+	testKongState(t, client, true, false, utils.KongRawState{}, nil)
 }