Merge pull request kubernetes#130162 from carlory/remove-cloud-provid…

…er-and-cloud-config-in-apiserver-CLI

kube-apiserver: removed the deprecated the `--cloud-provider` and `--cloud-config` CLI parameters.

cluster/gce/gci/configure-kubeapiserver.sh

@@ -55,9 +55,6 @@ function configure-etcd-params {
 # in the manifest file, and then copies the manifest file to /etc/kubernetes/manifests.
 #
 # Assumed vars (which are calculated in function compute-master-manifest-variables)
-#   CLOUD_CONFIG_OPT
-#   CLOUD_CONFIG_VOLUME
-#   CLOUD_CONFIG_MOUNT
 #   DOCKER_REGISTRY
 #   INSECURE_PORT_MAPPING
 function start-kube-apiserver {
@@ -66,9 +63,8 @@ function start-kube-apiserver {
   prepare-log-file "${KUBE_API_SERVER_AUDIT_LOG_PATH:-/var/log/kube-apiserver-audit.log}" "${KUBE_API_SERVER_RUNASUSER:-0}"
 
   # Calculate variables and assemble the command line.
-  local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-} ${CLOUD_CONFIG_OPT}"
+  local params="${API_SERVER_TEST_LOG_LEVEL:-"--v=2"} ${APISERVER_TEST_ARGS:-}"
   params+=" --allow-privileged=true"
-  params+=" --cloud-provider=${CLOUD_PROVIDER_FLAG:-external}"
   params+=" --client-ca-file=${CA_CERT_BUNDLE_PATH}"
 
   # params is passed by reference, so no "$"
@@ -384,16 +380,12 @@ function start-kube-apiserver {
   sed -i -e "s@{{params}}@${params}@g" "${src_file}"
   sed -i -e "s@{{container_env}}@${container_env}@g" "${src_file}"
   sed -i -e "s@{{srv_sshproxy_path}}@/etc/srv/sshproxy@g" "${src_file}"
-  sed -i -e "s@{{cloud_config_mount}}@${CLOUD_CONFIG_MOUNT}@g" "${src_file}"
-  sed -i -e "s@{{cloud_config_volume}}@${CLOUD_CONFIG_VOLUME}@g" "${src_file}"
   sed -i -e "s@{{pillar\['kube_docker_registry'\]}}@${DOCKER_REGISTRY}@g" "${src_file}"
   sed -i -e "s@{{pillar\['kube-apiserver_docker_tag'\]}}@${kube_apiserver_docker_tag}@g" "${src_file}"
   sed -i -e "s@{{pillar\['allow_privileged'\]}}@true@g" "${src_file}"
   sed -i -e "s@{{liveness_probe_initial_delay}}@${KUBE_APISERVER_LIVENESS_PROBE_INITIAL_DELAY_SEC:-15}@g" "${src_file}"
   sed -i -e "s@{{secure_port}}@443@g" "${src_file}"
   sed -i -e "s@{{insecure_port_mapping}}@${INSECURE_PORT_MAPPING}@g" "${src_file}"
-  sed -i -e "s@{{additional_cloud_config_mount}}@@g" "${src_file}"
-  sed -i -e "s@{{additional_cloud_config_volume}}@@g" "${src_file}"
   sed -i -e "s@{{webhook_authn_config_mount}}@${webhook_authn_config_mount}@g" "${src_file}"
   sed -i -e "s@{{webhook_authn_config_volume}}@${webhook_authn_config_volume}@g" "${src_file}"
   sed -i -e "s@{{webhook_config_mount}}@${webhook_config_mount}@g" "${src_file}"

cluster/gce/manifests/kube-apiserver.manifest

@@ -67,8 +67,6 @@
     "volumeMounts": [
         {{kms_socket_mount}}
         {{encryption_provider_mount}}
-        {{cloud_config_mount}}
-        {{additional_cloud_config_mount}}
         {{webhook_config_mount}}
         {{webhook_authn_config_mount}}
         {{csc_config_mount}}
@@ -108,8 +106,6 @@
 "volumes":[
   {{kms_socket_volume}}
   {{encryption_provider_volume}}
-  {{cloud_config_volume}}
-  {{additional_cloud_config_volume}}
   {{webhook_config_volume}}
   {{webhook_authn_config_volume}}
   {{csc_config_volume}}

cmd/kube-apiserver/app/options/completion.go

@@ -28,13 +28,11 @@ import (
 
 	cp "k8s.io/kubernetes/pkg/controlplane/apiserver/options"
 	"k8s.io/kubernetes/pkg/kubeapiserver"
-	kubeoptions "k8s.io/kubernetes/pkg/kubeapiserver/options"
 )
 
 // completedOptions is a private wrapper that enforces a call of Complete() before Run can be invoked.
 type completedOptions struct {
 	cp.CompletedOptions
-	CloudProvider *kubeoptions.CloudProviderOptions
 
 	Extra
 }
@@ -64,7 +62,6 @@ func (s *ServerRunOptions) Complete(ctx context.Context) (CompletedOptions, erro
 
 	completed := completedOptions{
 		CompletedOptions: controlplane,
-		CloudProvider:    s.CloudProvider,
 
 		Extra: s.Extra,
 	}

cmd/kube-apiserver/app/options/options.go

@@ -38,7 +38,6 @@ import (
 // ServerRunOptions runs a kubernetes api server.
 type ServerRunOptions struct {
 	*controlplaneapiserver.Options // embedded to avoid noise in existing consumers
-	CloudProvider                  *kubeoptions.CloudProviderOptions
 
 	Extra
 }
@@ -66,8 +65,7 @@ type Extra struct {
 // NewServerRunOptions creates and returns ServerRunOptions according to the given featureGate and effectiveVersion of the server binary to run.
 func NewServerRunOptions() *ServerRunOptions {
 	s := ServerRunOptions{
-		Options:       controlplaneapiserver.NewOptions(),
-		CloudProvider: kubeoptions.NewCloudProviderOptions(),
+		Options: controlplaneapiserver.NewOptions(),
 
 		Extra: Extra{
 			EndpointReconcilerType: string(reconcilers.LeaseEndpointReconcilerType),
@@ -101,7 +99,6 @@ func NewServerRunOptions() *ServerRunOptions {
 // Flags returns flags for a specific APIServer by section name
 func (s *ServerRunOptions) Flags() (fss cliflag.NamedFlagSets) {
 	s.Options.AddFlags(&fss)
-	s.CloudProvider.AddFlags(fss.FlagSet("cloud provider"))
 
 	// Note: the weird ""+ in below lines seems to be the only way to get gofmt to
 	// arrange these text blocks sensibly. Grrr.

cmd/kube-apiserver/app/options/options_test.go

@@ -104,8 +104,6 @@ func TestAddFlags(t *testing.T) {
 		"--authorization-webhook-config-file=/webhook-config",
 		"--bind-address=192.168.10.20",
 		"--client-ca-file=/client-ca",
-		"--cloud-config=/cloud-config",
-		"--cloud-provider=azure",
 		"--cors-allowed-origins=10.10.10.100,10.10.10.200",
 		"--contention-profiling=true",
 		"--egress-selector-config-file=/var/run/kubernetes/egress-selector/connectivity.yaml",
@@ -331,10 +329,6 @@ func TestAddFlags(t *testing.T) {
 			},
 			MasterCount: 5,
 		},
-		CloudProvider: &kubeoptions.CloudProviderOptions{
-			CloudConfigFile: "/cloud-config",
-			CloudProvider:   "azure",
-		},
 	}
 
 	expected.Authentication.OIDC.UsernameClaim = "sub"

cmd/kube-apiserver/app/options/validation.go

@@ -132,7 +132,6 @@ func (s CompletedOptions) Validate() []error {
 	var errs []error
 
 	errs = append(errs, s.CompletedOptions.Validate()...)
-	errs = append(errs, s.CloudProvider.Validate()...)
 	errs = append(errs, validateClusterIPFlags(s.Extra)...)
 	errs = append(errs, validateServiceNodePort(s.Extra)...)
 	errs = append(errs, validatePublicIPServiceClusterIPRangeIPFamilies(s.Extra, *s.GenericServerRunOptions)...)

cmd/kube-apiserver/app/server.go

@@ -212,9 +212,7 @@ func CreateKubeAPIServerConfig(
 	capabilities.Setup(opts.AllowPrivileged, opts.MaxConnectionBytesPerSec)
 
 	// additional admission initializers
-	kubeAdmissionConfig := &kubeapiserveradmission.Config{
-		CloudConfigFile: opts.CloudProvider.CloudConfigFile,
-	}
+	kubeAdmissionConfig := &kubeapiserveradmission.Config{}
 	kubeInitializers, err := kubeAdmissionConfig.New()
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to create admission plugin initializer: %w", err)

pkg/features/kube_features.go

@@ -175,16 +175,6 @@ const (
 	// both allocators. This feature gate disables the dual write on the new Cluster IP allocators.
 	DisableAllocatorDualWrite featuregate.Feature = "DisableAllocatorDualWrite"
 
-	// owner: @andrewsykim
-	//
-	// Disable any functionality in kube-apiserver, kube-controller-manager and kubelet related to the `--cloud-provider` component flag.
-	DisableCloudProviders featuregate.Feature = "DisableCloudProviders"
-
-	// owner: @andrewsykim
-	//
-	// Disable in-tree functionality in kubelet to authenticate to cloud provider container registries for image pull credentials.
-	DisableKubeletCloudCredentialProviders featuregate.Feature = "DisableKubeletCloudCredentialProviders"
-
 	// owner: @micahhausler
 	//
 	// Setting AllowInsecureKubeletCertificateSigningRequests to true disables node admission validation of CSRs

pkg/features/versioned_kube_features.go

@@ -167,18 +167,6 @@ var defaultVersionedKubernetesFeatureGates = map[featuregate.Feature]featuregate
 		{Version: version.MustParse("1.33"), Default: false, PreRelease: featuregate.Beta}, // remove after MultiCIDRServiceAllocator is GA
 	},
 
-	DisableCloudProviders: {
-		{Version: version.MustParse("1.22"), Default: false, PreRelease: featuregate.Alpha},
-		{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
-		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
-	},
-
-	DisableKubeletCloudCredentialProviders: {
-		{Version: version.MustParse("1.23"), Default: false, PreRelease: featuregate.Alpha},
-		{Version: version.MustParse("1.29"), Default: true, PreRelease: featuregate.Beta},
-		{Version: version.MustParse("1.31"), Default: true, PreRelease: featuregate.GA, LockToDefault: true},
-	},
-
 	DisableNodeKubeProxyVersion: {
 		{Version: version.MustParse("1.29"), Default: false, PreRelease: featuregate.Alpha},
 		{Version: version.MustParse("1.31"), Default: false, PreRelease: featuregate.Deprecated},

pkg/kubeapiserver/admission/config.go

@@ -17,28 +17,13 @@ limitations under the License.
 package admission
 
 import (
-	"os"
-
-	"k8s.io/klog/v2"
-
 	"k8s.io/apiserver/pkg/admission"
 )
 
 // Config holds the configuration needed to for initialize the admission plugins
-type Config struct {
-	CloudConfigFile string
-}
+type Config struct{}
 
 // New sets up the plugins and admission start hooks needed for admission
 func (c *Config) New() ([]admission.PluginInitializer, error) {
-	var cloudConfig []byte
-	if c.CloudConfigFile != "" {
-		var err error
-		cloudConfig, err = os.ReadFile(c.CloudConfigFile)
-		if err != nil {
-			klog.Fatalf("Error reading from cloud configuration file %s: %#v", c.CloudConfigFile, err)
-		}
-	}
-
-	return []admission.PluginInitializer{NewPluginInitializer(cloudConfig)}, nil
+	return []admission.PluginInitializer{NewPluginInitializer()}, nil
 }

pkg/kubeapiserver/admission/initializer.go

@@ -22,29 +22,17 @@ import (
 
 // TODO add a `WantsToRun` which takes a stopCh.  Might make it generic.
 
-// WantsCloudConfig defines a function which sets CloudConfig for admission plugins that need it.
-type WantsCloudConfig interface {
-	SetCloudConfig([]byte)
-}
-
 // PluginInitializer is used for initialization of the Kubernetes specific admission plugins.
 type PluginInitializer struct {
-	cloudConfig []byte
 }
 
 var _ admission.PluginInitializer = &PluginInitializer{}
 
 // NewPluginInitializer constructs new instance of PluginInitializer
-func NewPluginInitializer(cloudConfig []byte) *PluginInitializer {
-	return &PluginInitializer{
-		cloudConfig: cloudConfig,
-	}
+func NewPluginInitializer() *PluginInitializer {
+	return &PluginInitializer{}
 }
 
 // Initialize checks the initialization interfaces implemented by each plugin
 // and provide the appropriate initialization data
-func (i *PluginInitializer) Initialize(plugin admission.Interface) {
-	if wants, ok := plugin.(WantsCloudConfig); ok {
-		wants.SetCloudConfig(i.cloudConfig)
-	}
-}
+func (i *PluginInitializer) Initialize(plugin admission.Interface) {}

test/e2e_node/kubelet_config_dir_test.go

@@ -86,7 +86,6 @@ shutdownGracePeriodByPodPriority:
   - priority: 3
     shutdownGracePeriodSeconds: 30
 featureGates:
-  DisableKubeletCloudCredentialProviders: true
   PodAndContainerStatsFromCRI: true`)
 			framework.ExpectNoError(os.WriteFile(filepath.Join(configDir, "10-kubelet.conf"), contents, 0755))
 			contents = []byte(`apiVersion: kubelet.config.k8s.io/v1beta1
@@ -164,7 +163,7 @@ featureGates:
 				},
 			}
 			// This covers the case where the fields within the map are overridden.
-			overrides := map[string]bool{"DisableKubeletCloudCredentialProviders": true, "PodAndContainerStatsFromCRI": false, "DynamicResourceAllocation": true}
+			overrides := map[string]bool{"PodAndContainerStatsFromCRI": false, "DynamicResourceAllocation": true}
 			// In some CI jobs, `NodeSwap` is explicitly disabled as the images are cgroupv1 based,
 			// so such flags should be picked up directly from the initial configuration
 			if _, ok := initialConfig.FeatureGates["NodeSwap"]; ok {

test/e2e_node/remote/node_e2e.go

@@ -97,10 +97,9 @@ func prependMemcgNotificationFlag(args string) string {
 // a credential provider plugin.
 func prependCredentialProviderFlag(args, workspace string) string {
 	credentialProviderConfig := filepath.Join(workspace, "credential-provider.yaml")
-	featureGateFlag := "--kubelet-flags=--feature-gates=DisableKubeletCloudCredentialProviders=true"
 	configFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-config=%s", credentialProviderConfig)
 	binFlag := fmt.Sprintf("--kubelet-flags=--image-credential-provider-bin-dir=%s", workspace)
-	return fmt.Sprintf("%s %s %s %s", featureGateFlag, configFlag, binFlag, args)
+	return fmt.Sprintf("%s %s %s", configFlag, binFlag, args)
 }
 
 // osSpecificActions takes OS specific actions required for the node tests