✨ Allow more permissive extensibility for securityRules and additional CP LoadBalancers

api/v1beta1/types.go

@@ -111,9 +111,22 @@
 	// +optional
 	ControlPlaneOutboundLB *LoadBalancerSpec `json:"controlPlaneOutboundLB,omitempty"`
 
+	// AdditionalControlPlaneLBPorts is the configuration for the additional inbound control-plane load balancer ports
+	// +optional
+	AdditionalControlPlaneLBPorts []LoadBalancerPort `json:"additionalControlPlaneLBPorts,omitempty"`
+
 	NetworkClassSpec `json:",inline"`
 }
 
+// LoadBalancerPort specifies additional port for the API server load balancer.
+type LoadBalancerPort struct {
+	// Name for the additional port within LB definition
+	Name string `json:"name"`
+
+	// Port for the LB definition
+	Port int32 `json:"port"`
+}
+
 // VnetSpec configures an Azure virtual network.
 type VnetSpec struct {
 	// ResourceGroup is the name of the resource group of the existing virtual network
@@ -893,6 +906,17 @@
 	return false
 }
 
+// GetSecurityRuleByDestination returns security group rule, which matches provided destination ports.
+func (s SubnetSpec) GetSecurityRuleByDestination(ports string) *SecurityRule {
+	for _, rule := range s.SecurityGroup.SecurityRules {
+		if rule.DestinationPorts != nil && *rule.DestinationPorts == ports {
+			return &rule
+		}
+	}
+
+	return nil
+}
+
 // SecurityProfile specifies the Security profile settings for a
 // virtual machine or virtual machine scale set.
 type SecurityProfile struct {

api/v1beta1/types_template.go

@@ -75,6 +75,10 @@ type NetworkTemplateSpec struct {
 	// This is different from APIServerLB, and is used only in private clusters (optionally) for enabling outbound traffic.
 	// +optional
 	ControlPlaneOutboundLB *LoadBalancerClassSpec `json:"controlPlaneOutboundLB,omitempty"`
+
+	// AdditionalControlPlaneLBPorts is the configuration for the additional inbound control-plane load balancer ports
+	// +optional
+	AdditionalControlPlaneLBPorts []LoadBalancerPort `json:"additionalControlPlaneLBPorts,omitempty"`
 }
 
 // GetSubnetTemplate returns the subnet template based on the subnet role.

azure/scope/cluster.go

@@ -266,6 +266,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
 			BackendPoolName:      s.APIServerLB().BackendPool.Name,
 			IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes,
 			AdditionalTags:       s.AdditionalTags(),
+			AdditionalPorts:      s.ControlPlaneAdditionalLBPorts(),
 		}
 
 		if s.APIServerLB().FrontendIPs != nil {
@@ -299,6 +300,7 @@ func (s *ClusterScope) LBSpecs() []azure.ResourceSpecGetter {
 			BackendPoolName:      s.APIServerLB().BackendPool.Name + "-internal",
 			IdleTimeoutInMinutes: s.APIServerLB().IdleTimeoutInMinutes,
 			AdditionalTags:       s.AdditionalTags(),
+			AdditionalPorts:      s.ControlPlaneAdditionalLBPorts(),
 		}
 
 		privateIPFound := false
@@ -771,6 +773,11 @@ func (s *ClusterScope) ControlPlaneOutboundLB() *infrav1.LoadBalancerSpec {
 	return s.AzureCluster.Spec.NetworkSpec.ControlPlaneOutboundLB
 }
 
+// ControlPlaneAdditionalLBPorts returns the additional API server ports list.
+func (s *ClusterScope) ControlPlaneAdditionalLBPorts() []infrav1.LoadBalancerPort {
+	return s.AzureCluster.Spec.NetworkSpec.AdditionalControlPlaneLBPorts
+}
+
 // APIServerLBName returns the API Server LB name.
 func (s *ClusterScope) APIServerLBName() string {
 	apiServerLB := s.APIServerLB()
@@ -1020,9 +1027,15 @@ func (s *ClusterScope) SetControlPlaneSecurityRules() {
 	if !s.ControlPlaneEnabled() {
 		return
 	}
-	if s.ControlPlaneSubnet().SecurityGroup.SecurityRules == nil {
-		subnet := s.ControlPlaneSubnet()
-		subnet.SecurityGroup.SecurityRules = infrav1.SecurityRules{
+
+	subnet := s.ControlPlaneSubnet()
+
+	if subnet.SecurityGroup.SecurityRules == nil {
+		s.AzureCluster.Spec.NetworkSpec.UpdateControlPlaneSubnet(subnet)
+	}
+
+	if subnet.GetSecurityRuleByDestination("22") == nil {
+		subnet.SecurityGroup.SecurityRules = append(s.ControlPlaneSubnet().SecurityGroup.SecurityRules,
 			infrav1.SecurityRule{
 				Name:             "allow_ssh",
 				Description:      "Allow SSH",
@@ -1034,20 +1047,26 @@ func (s *ClusterScope) SetControlPlaneSecurityRules() {
 				Destination:      ptr.To("*"),
 				DestinationPorts: ptr.To("22"),
 				Action:           infrav1.SecurityRuleActionAllow,
-			},
-			infrav1.SecurityRule{
-				Name:             "allow_apiserver",
-				Description:      "Allow K8s API Server",
-				Priority:         2201,
-				Protocol:         infrav1.SecurityGroupProtocolTCP,
-				Direction:        infrav1.SecurityRuleDirectionInbound,
-				Source:           ptr.To("*"),
-				SourcePorts:      ptr.To("*"),
-				Destination:      ptr.To("*"),
-				DestinationPorts: ptr.To(strconv.Itoa(int(s.APIServerPort()))),
-				Action:           infrav1.SecurityRuleActionAllow,
-			},
-		}
+			})
+
+		s.AzureCluster.Spec.NetworkSpec.UpdateControlPlaneSubnet(subnet)
+	}
+
+	port := strconv.Itoa(int(s.APIServerPort()))
+	if subnet.GetSecurityRuleByDestination(port) == nil {
+		subnet.SecurityGroup.SecurityRules = append(s.ControlPlaneSubnet().SecurityGroup.SecurityRules, infrav1.SecurityRule{
+			Name:             "allow_apiserver",
+			Description:      "Allow K8s API Server",
+			Priority:         2201,
+			Protocol:         infrav1.SecurityGroupProtocolTCP,
+			Direction:        infrav1.SecurityRuleDirectionInbound,
+			Source:           ptr.To("*"),
+			SourcePorts:      ptr.To("*"),
+			Destination:      ptr.To("*"),
+			DestinationPorts: ptr.To(port),
+			Action:           infrav1.SecurityRuleActionAllow,
+		})
+
 		s.AzureCluster.Spec.NetworkSpec.UpdateControlPlaneSubnet(subnet)
 	}
 }

azure/services/loadbalancers/spec.go

@@ -47,6 +47,7 @@
 	APIServerPort        int32
 	IdleTimeoutInMinutes *int32
 	AdditionalTags       map[string]string
+	AdditionalPorts      []infrav1.LoadBalancerPort
 }
 
 // ResourceName returns the name of the load balancer.
@@ -221,14 +222,14 @@
 		if len(frontendIDs) != 0 {
 			frontendIPConfig = frontendIDs[0]
 		}
-		return []*armnetwork.LoadBalancingRule{
+		rules := []*armnetwork.LoadBalancingRule{
 			{
 				Name: ptr.To(lbRuleHTTPS),
 				Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
 					DisableOutboundSnat:     ptr.To(true),
 					Protocol:                ptr.To(armnetwork.TransportProtocolTCP),
-					FrontendPort:            ptr.To[int32](lbSpec.APIServerPort),
-					BackendPort:             ptr.To[int32](lbSpec.APIServerPort),
+					FrontendPort:            ptr.To(lbSpec.APIServerPort),
+					BackendPort:             ptr.To(lbSpec.APIServerPort),
 					IdleTimeoutInMinutes:    lbSpec.IdleTimeoutInMinutes,
 					EnableFloatingIP:        ptr.To(false),
 					LoadDistribution:        ptr.To(armnetwork.LoadDistributionDefault),
@@ -242,6 +243,30 @@
 				},
 			},
 		}
+
+		for _, port := range lbSpec.AdditionalPorts {
+			rules = append(rules, &armnetwork.LoadBalancingRule{
+				Name: ptr.To(port.Name),
+				Properties: &armnetwork.LoadBalancingRulePropertiesFormat{
+					DisableOutboundSnat:     ptr.To(true),
+					Protocol:                ptr.To(armnetwork.TransportProtocolTCP),
+					FrontendPort:            ptr.To(port.Port),
+					BackendPort:             ptr.To(port.Port),
+					IdleTimeoutInMinutes:    lbSpec.IdleTimeoutInMinutes,
+					EnableFloatingIP:        ptr.To(false),
+					LoadDistribution:        ptr.To(armnetwork.LoadDistributionDefault),
+					FrontendIPConfiguration: frontendIPConfig,
+					BackendAddressPool: &armnetwork.SubResource{
+						ID: ptr.To(azure.AddressPoolID(lbSpec.SubscriptionID, lbSpec.ResourceGroup, lbSpec.Name, lbSpec.BackendPoolName)),
+					},
+					Probe: &armnetwork.SubResource{
+						ID: ptr.To(azure.ProbeID(lbSpec.SubscriptionID, lbSpec.ResourceGroup, lbSpec.Name, httpsProbe)),
+					},
+				},
+			})
+		}
+
+		return rules
 	}
 	return []*armnetwork.LoadBalancingRule{}
 }

config/crd/bases/infrastructure.cluster.x-k8s.io_azureclusters.yaml

@@ -658,6 +658,25 @@ spec:
                 description: NetworkSpec encapsulates all things related to Azure
                   network.
                 properties:
+                  additionalControlPlaneLBPorts:
+                    description: AdditionalControlPlaneLBPorts is the configuration
+                      for the additional inbound control-plane load balancer ports
+                    items:
+                      description: LoadBalancerPort specifies additional port for
+                        the API server load balancer.
+                      properties:
+                        name:
+                          description: Name for the additional port within LB definition
+                          type: string
+                        port:
+                          description: Port for the LB definition
+                          format: int32
+                          type: integer
+                      required:
+                      - name
+                      - port
+                      type: object
+                    type: array
                   apiServerLB:
                     description: APIServerLB is the configuration for the control-plane
                       load balancer.

config/crd/bases/infrastructure.cluster.x-k8s.io_azureclustertemplates.yaml

@@ -520,6 +520,27 @@ spec:
                         description: NetworkSpec encapsulates all things related to
                           Azure network.
                         properties:
+                          additionalControlPlaneLBPorts:
+                            description: AdditionalControlPlaneLBPorts is the configuration
+                              for the additional inbound control-plane load balancer
+                              ports
+                            items:
+                              description: LoadBalancerPort specifies additional port
+                                for the API server load balancer.
+                              properties:
+                                name:
+                                  description: Name for the additional port within
+                                    LB definition
+                                  type: string
+                                port:
+                                  description: Port for the LB definition
+                                  format: int32
+                                  type: integer
+                              required:
+                              - name
+                              - port
+                              type: object
+                            type: array
                           apiServerLB:
                             description: APIServerLB is the configuration for the
                               control-plane load balancer.

docs/book/src/self-managed/custom-vnet.md

@@ -114,8 +114,13 @@ Security Rules were previously known as `ingressRule` in v1alpha3.
 </aside>
 
 Security rules can also be customized as part of the subnet specification in a custom network spec.
-Note that ingress rules for the Kubernetes API Server port (default 6443) and SSH (22) are automatically added to the controlplane subnet only if security rules aren't specified.
-It is the responsibility of the user to supply those rules themselves if using custom rules.
+
+Note that ingress rules for the Kubernetes API Server port (default 6443) and SSH (22) are automatically added to the controlplane subnet if these security rules aren't specified.
+It is the responsibility of the user to override those rules themselves when the default configuration does not match expected ruleset.
+
+These rules are identified by `destinationPorts` value:
+- `<API_SERVER_PORT>` for the API server access. Default port is `6443`.
+- `22` for the SSH access.
 
 Here is an illustrative example of customizing rules that builds on the one above by adding an egress rule to the control plane nodes:
 
@@ -141,22 +146,22 @@ spec:
           name: my-subnet-cp-nsg
           securityRules:
             - name: "allow_ssh"
-              description: "allow SSH"
+              description: "Deny SSH"
               direction: "Inbound"
               priority: 2200
               protocol: "*"
               destination: "*"
               destinationPorts: "22"
               source: "*"
               sourcePorts: "*"
-              action: "Allow"
+              action: "Deny"
             - name: "allow_apiserver"
-              description: "Allow K8s API Server"
+              description: "Allow Custom K8s API Server"
               direction: "Inbound"
               priority: 2201
               protocol: "*"
               destination: "*"
-              destinationPorts: "6443"
+              destinationPorts: "1234" # Custom API server URL
               source: "*"
               sourcePorts: "*"
               action: "Allow"
@@ -177,6 +182,49 @@ spec:
   resourceGroup: cluster-example
 ```
 
+Alternatively, when default server `securityRules` apply, but the list needs to be extended, only required rules can be added, like so:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: AzureCluster
+metadata:
+  name: cluster-example
+  namespace: default
+spec:
+  location: southcentralus
+  networkSpec:
+    vnet:
+      name: my-vnet
+      cidrBlocks:
+        - 10.0.0.0/16
+    additionalControlPlaneLBPorts:
+      - name: RKE2
+        port: 9345
+    subnets:
+      - name: my-subnet-cp
+        role: control-plane
+        cidrBlocks:
+          - 10.0.1.0/24
+        securityGroup:
+          name: my-subnet-cp-nsg
+          securityRules:
+            - name: "allow_port_9345"
+              description: "RKE2 - allow node registration on port 9345"
+              direction: "Inbound"
+              priority: 2202
+              protocol: "Tcp"
+              destination: "*"
+              destinationPorts: "9345"
+              source: "*"
+              sourcePorts: "*"
+              action: "Allow"
+      - name: my-subnet-node
+        role: node
+        cidrBlocks:
+          - 10.0.2.0/24
+  resourceGroup: cluster-example
+```
+
 ### Virtual Network service endpoints
 
 Sometimes it's desirable to use [Virtual Network service endpoints](https://learn.microsoft.com/azure/virtual-network/virtual-network-service-endpoints-overview) to establish secure and direct connectivity to Azure services from your subnet(s). Service Endpoints are configured on a per-subnet basis. Vnets managed by either `AzureCluster` or `AzureManagedControlPlane` can have `serviceEndpoints` optionally set on each subnet.