Add Kubernetes Client QPS and Burst Configuration

chart/templates/deployment.yaml

@@ -98,6 +98,12 @@ spec:
         {{- with .Values.controller.manager.extraArgs }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
+        {{- if .Values.controller.manager.kubeClient.qps }}
+        - --kube-client-qps={{ .Values.controller.manager.kubeClient.qps }}
+        {{- end }}
+        {{- if .Values.controller.manager.kubeClient.burst }}
+        - --kube-client-burst={{ .Values.controller.manager.kubeClient.burst }}
+        {{- end }}
         command:
         - /vault-secrets-operator
         env:

chart/values.yaml

@@ -156,7 +156,6 @@ controller:
 
   # Settings related to the vault-secrets-operator container.
   manager:
-
     # Image sets the repo and tag of the vault-secrets-operator image to use for the controller.
     image:
       pullPolicy: IfNotPresent
@@ -465,6 +464,21 @@ controller:
     # @type: integer
     maxConcurrentReconciles:
 
+    kubeClient:
+      # QPS indicates the maximum QPS to the kubernetes API.
+      # When the value is 0, the kubernetes client's default is used.
+      # May also set via the `VSO_KUBE_CLIENT_QPS` environment variable.
+      # Default: 0
+      # @type: float
+      qps:
+
+      # Maximum burst for throttling requests to the kubernetes API.
+      # When the value is 0, the kubernetes client's default is used.
+      # May also set via the `VSO_KUBE_CLIENT_BURST` environment variable.
+      # Default: 0
+      # @type: uint
+      burst:
+
     # Defines additional environment variables to be added to the
     # vault-secrets-operator manager container.
     # Example:
@@ -552,10 +566,10 @@ metricsService:
   # https://kubernetes.io/docs/concepts/services-networking/service/
   # @type: map
   ports:
-  - name: https
-    port: 8443
-    protocol: TCP
-    targetPort: https
+    - name: https
+      port: 8443
+      protocol: TCP
+      targetPort: https
   type: ClusterIP
 
 # Configures the default VaultConnection CR which will be used by resources
@@ -593,7 +607,6 @@ defaultVaultConnection:
   # @type: map
   headers: {}
 
-
 # Configures and deploys the default VaultAuthMethod CR which will be used by resources
 # if they do not specify a VaultAuthMethod reference. The name is 'default' and will
 # always be installed in the same namespace as the operator.
@@ -791,7 +804,6 @@ defaultAuthMethod:
       # @type: string
       params: none
 
-
 # Configures a Prometheus ServiceMonitor
 telemetry:
   serviceMonitor:
@@ -859,7 +871,7 @@ hooks:
 
     # Limit the number of retries for the CRD upgrade.
     # @type: integer
-    backoffLimit:  5
+    backoffLimit: 5
 
     # Set the timeout for the CRD upgrade. The operation should typically take less than 5s
     # to complete.

internal/options/env.go

@@ -49,6 +49,12 @@ type VSOEnvOptions struct {
 
 	// ClientCacheNumLocks is VSO_CLIENT_CACHE_NUM_LOCKS environment variable option
 	ClientCacheNumLocks *int `split_words:"true"`
+
+	// KubeClientQPS is the VSO_KUBE_CLIENT_QPS environment variable option
+	KubeClientQPS float64 `split_words:"true"`
+
+	// KubeClientBurst is the VSO_KUBE_CLIENT_BURST environment variable option
+	KubeClientBurst *uint `split_words:"true"`
 }
 
 // Parse environment variable options, prefixed with "VSO_"

internal/options/env_test.go

@@ -35,6 +35,8 @@ func TestParse(t *testing.T) {
 				"VSO_GLOBAL_TRANSFORMATION_OPTIONS":  "gOpt1,gOpt2",
 				"VSO_GLOBAL_VAULT_AUTH_OPTIONS":      "vOpt1,vOpt2",
 				"VSO_CLIENT_CACHE_NUM_LOCKS":         "10",
+				"VSO_KUBE_CLIENT_QPS":                "100",
+				"VSO_KUBE_CLIENT_BURST":              "1000",
 			},
 			wantOptions: VSOEnvOptions{
 				OutputFormat:                "json",
@@ -49,6 +51,8 @@ func TestParse(t *testing.T) {
 				GlobalTransformationOptions: []string{"gOpt1", "gOpt2"},
 				GlobalVaultAuthOptions:      []string{"vOpt1", "vOpt2"},
 				ClientCacheNumLocks:         ptr.To(10),
+				KubeClientQPS:               100,
+				KubeClientBurst:             ptr.To(uint(1000)),
 			},
 		},
 	}

main.go

@@ -147,6 +147,8 @@ func main() {
 	var backoffRandomizationFactor float64
 	var backoffMultiplier float64
 	var backoffMaxElapsedTime time.Duration
+	var kubeClientQPS float64
+	var kubeClientBurst uint
 
 	// command-line args and flags
 	flag.BoolVar(&printVersion, "version", false, "Print the operator version information")
@@ -216,6 +218,14 @@ func main() {
 			"All errors are tried using an exponential backoff strategy. "+
 			"The value must be greater than zero. "+
 			"Also set from environment variable VSO_BACKOFF_MULTIPLIER.")
+	flag.Float64Var(&kubeClientQPS, "kube-client-qps", 0,
+		"Maximum queries per second to limit requests sent to the API server and prevent overload. "+
+			"When the value is 0, the kubernetes client's default is used. "+
+			"Also set from environment variable VSO_KUBE_CLIENT_QPS.")
+	flag.UintVar(&kubeClientBurst, "kube-client-burst", 0,
+		"Maximum burst for throttling requests to the Kubernetes API. "+
+			"When the value is 0, the kubernetes client's default is used. "+
+			"Also set from environment variable VSO_KUBE_CLIENT_BURST.")
 
 	opts := zap.Options{
 		Development: os.Getenv("VSO_LOGGER_DEVELOPMENT_MODE") != "",
@@ -269,6 +279,12 @@ func main() {
 	} else if globalVaultAuthOpts != "" {
 		globalVaultAuthOptsSet = strings.Split(globalVaultAuthOpts, ",")
 	}
+	if vsoEnvOptions.KubeClientQPS != 0 {
+		kubeClientQPS = vsoEnvOptions.KubeClientQPS
+	}
+	if vsoEnvOptions.KubeClientBurst != nil {
+		kubeClientBurst = *vsoEnvOptions.KubeClientBurst
+	}
 
 	// versionInfo is used when setting up the buildInfo metric below
 	versionInfo := version.Version()
@@ -342,6 +358,13 @@ func main() {
 	cfc.GlobalVaultAuthOptions = globalVaultAuthOptions
 
 	config := ctrl.GetConfigOrDie()
+	// set the Kube Client QPS and Burst config if they are set
+	if kubeClientQPS != 0 {
+		config.QPS = float32(kubeClientQPS)
+	}
+	if kubeClientBurst != 0 {
+		config.Burst = int(kubeClientBurst)
+	}
 
 	defaultClient, err := client.NewWithWatch(config, client.Options{
 		Scheme: scheme,

test/unit/deployment.bats

@@ -1292,3 +1292,61 @@ load _helpers
   [ "$(echo "${job}" | \
   yq '(.spec.template.spec.containers[0].imagePullPolicy == "IfNotPresent")')" = "true" ]
 }
+
+
+#--------------------------------------------------------------------
+# kubeClient
+
+@test "controller/Deployment: kubeClient qps not set by default" {
+  cd `chart_dir`
+  local object
+  object=$(helm template \
+  -s templates/deployment.yaml  \
+  . | tee /dev/stderr |
+  yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo "$object" | yq 'contains(["--kube-client-qps"])' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "controller/Deployment: kubeClient qps can be set" {
+  cd `chart_dir`
+  local object
+  object=$(helm template \
+  -s templates/deployment.yaml  \
+  --set 'controller.manager.kubeClient.qps=200' \
+  . | tee /dev/stderr |
+  yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo "$object" | yq 'contains(["--kube-client-qps=200"])' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}
+
+@test "controller/Deployment: kubeClient burst not set by default" {
+  cd `chart_dir`
+  local object
+  object=$(helm template \
+  -s templates/deployment.yaml  \
+  . | tee /dev/stderr |
+  yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo "$object" | yq 'contains(["--kube-client-burst"])' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "controller/Deployment: kubeClient burst can be set" {
+  cd `chart_dir`
+  local object
+  object=$(helm template \
+  -s templates/deployment.yaml  \
+  --set 'controller.manager.kubeClient.burst=2000' \
+  . | tee /dev/stderr |
+  yq 'select(.kind == "Deployment" and .metadata.labels."control-plane" == "controller-manager") | .spec.template.spec.containers[] | select(.name == "manager") | .args' | tee /dev/stderr)
+
+  local actual
+  actual=$(echo "$object" | yq 'contains(["--kube-client-burst=2000"])' | tee /dev/stderr)
+  [ "${actual}" = "true" ]
+}