Bootstrap flag for connection limit overload manager resource monitor

apis/projectcontour/v1alpha1/contourconfig.go

@@ -323,6 +323,13 @@ type EnvoyConfig struct {
 	// Network holds various configurable Envoy network values.
 	// +optional
 	Network *NetworkParameters `json:"network,omitempty"`
+
+	// OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+	// the envoy overload manager actions, such as global connection limits, enforced.
+	//
+	// This is disabled by default
+	// +optional
+	OMEnforcedHealth *HealthConfig `json:"omEnforcedHealth,omitempty"`
 }
 
 // DebugConfig contains Contour specific troubleshooting options.

apis/projectcontour/v1alpha1/contourconfig_helpers.go

@@ -124,6 +124,10 @@ func (e *EnvoyConfig) Validate() error {
 		return fmt.Errorf("invalid envoy configuration: %v", err)
 	}
 
+	if err := healthEndpointsInConflict(e.Metrics, e.Health, e.OMEnforcedHealth); err != nil {
+		return fmt.Errorf("invalid envoy configuration: %v", err)
+	}
+
 	if err := e.Logging.Validate(); err != nil {
 		return err
 	}
@@ -342,3 +346,17 @@ func endpointsInConfict(health *HealthConfig, metrics *MetricsConfig) error {
 	}
 	return nil
 }
+
+// healthEndpointsInConflict returns an error if the same address and port are used between the overload manager enforced health listener and metrics
+// _or_ the stats listener. Since the metrics would be validated against the stats listener already we only need to check that the overload manager
+// health listener is not in conflict with either metrics or stats
+func healthEndpointsInConflict(metrics *MetricsConfig, stats, omEnforcedHealth *HealthConfig) error {
+	switch {
+	case omEnforcedHealth != nil && stats != nil && omEnforcedHealth.Address == stats.Address && omEnforcedHealth.Port == stats.Port:
+		return fmt.Errorf("cannot use the same port for health checks and overload-manager enforced health checks")
+	case omEnforcedHealth != nil && metrics != nil && omEnforcedHealth.Address == metrics.Address && omEnforcedHealth.Port == metrics.Port:
+		return fmt.Errorf("cannot use the same port for metrics and overload-manager enforced health checks")
+	default:
+		return nil
+	}
+}

changelogs/unreleased/6794-seth-epps-minor.md

@@ -0,0 +1,52 @@
+## Overload Manager - Max Global Connections
+
+Introduces an envoy bootstrap flag to enable the [global downstream connection limit overload manager resource monitors](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto#envoy-v3-api-msg-extensions-resource-monitors-downstream-connections-v3-downstreamconnectionsconfig).
+
+The new flag can be passed as an integer flag to the contour bootstrap subcommand, `overload-dowstream-max-conn`.
+
+```sh
+contour bootstrap --help
+INFO[0000] maxprocs: Leaving GOMAXPROCS=10: CPU quota undefined
+usage: contour bootstrap [<flags>] <path>
+
+Generate bootstrap configuration.
+
+
+Flags:
+  -h, --[no-]help                Show context-sensitive help (also try --help-long and --help-man).
+      --log-format=text          Log output format for Contour. Either text or json.
+      --admin-address="/admin/admin.sock"
+                                 Path to Envoy admin unix domain socket.
+      --admin-port=ADMIN-PORT    DEPRECATED: Envoy admin interface port.
+      --dns-lookup-family=DNS-LOOKUP-FAMILY
+                                 Defines what DNS Resolution Policy to use for Envoy -> Contour cluster name lookup. Either v4, v6, auto, or all.
+      --envoy-cafile=ENVOY-CAFILE
+                                 CA Filename for Envoy secure xDS gRPC communication. ($ENVOY_CAFILE)
+      --envoy-cert-file=ENVOY-CERT-FILE
+                                 Client certificate filename for Envoy secure xDS gRPC communication. ($ENVOY_CERT_FILE)
+      --envoy-key-file=ENVOY-KEY-FILE
+                                 Client key filename for Envoy secure xDS gRPC communication. ($ENVOY_KEY_FILE)
+      --namespace="projectcontour"
+                                 The namespace the Envoy container will run in. ($CONTOUR_NAMESPACE)
+      --overload-dowstream-max-conn=OVERLOAD-DOWSTREAM-MAX-CONN
+                                 Defines the Envoy global downstream connection limit
+      --overload-max-heap=OVERLOAD-MAX-HEAP
+                                 Defines the maximum heap size in bytes until overload manager stops accepting new connections.
+      --resources-dir=RESOURCES-DIR
+                                 Directory where configuration files will be written to.
+      --xds-address=XDS-ADDRESS  xDS gRPC API address.
+      --xds-port=XDS-PORT        xDS gRPC API port.
+      --xds-resource-version="v3"
+                                 The versions of the xDS resources to request from Contour.
+
+Args:
+  <path>  Configuration file ('-' for standard output).
+```
+
+As part of this change, we also set the `ignore_global_conn_limit` flag to `true` on the existing admin listeners such
+that envoy remains live, ready, and serving stats even though it is rejecting downstream connections.
+
+To add some flexibility for health checks, in addition to adding a new bootstrap flag, there is a new configuration
+option for the envoy health config to enforce the envoy overload manager actions, namely rejecting requests. This
+"advanced" configuration gives the operator the ability to configure readiness and liveness to handle taking pods out
+of the pool of pods that can serve traffic.

cmd/contour/bootstrap.go

@@ -34,6 +34,7 @@ func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.Boo
 	bootstrap.Flag("envoy-cert-file", "Client certificate filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
 	bootstrap.Flag("envoy-key-file", "Client key filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
 	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
+	bootstrap.Flag("overload-dowstream-max-conn", "Defines the Envoy global downstream connection limit.").Int64Var(&config.GlobalDownstreamConnectionLimit)
 	bootstrap.Flag("overload-max-heap", "Defines the maximum heap size in bytes until overload manager stops accepting new connections.").Uint64Var(&config.MaximumHeapSizeBytes)
 	bootstrap.Flag("resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir)
 	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)

cmd/contour/contour.go

@@ -104,6 +104,9 @@
 		if err := envoy.ValidAdminAddress(bootstrapCtx.AdminAddress); err != nil {
 			log.WithField("flag", "--admin-address").WithError(err).Fatal("failed to parse bootstrap args")
 		}
+		if err := envoy.ValidConnectionLimit(bootstrapCtx.GlobalDownstreamConnectionLimit); err != nil {
+			log.WithField("flag", "--overload-dowstream-max-conn").WithError(err).Fatal("failed to parse bootstrap args")
+		}
 		if err := envoy_v3.WriteBootstrap(bootstrapCtx); err != nil {
 			log.WithError(err).Fatal("failed to write bootstrap configuration")
 		}

cmd/contour/serve.go

@@ -494,7 +494,7 @@
 	}
 
 	resources := []xdscache.ResourceCache{
-		xdscache_v3.NewListenerCache(listenerConfig, *contourConfiguration.Envoy.Metrics, *contourConfiguration.Envoy.Health, *contourConfiguration.Envoy.Network.EnvoyAdminPort),
+		xdscache_v3.NewListenerCache(listenerConfig, *contourConfiguration.Envoy.Metrics, *contourConfiguration.Envoy.Health, *contourConfiguration.Envoy.Network.EnvoyAdminPort, contourConfiguration.Envoy.OMEnforcedHealth),
 		xdscache_v3.NewSecretsCache(envoy_v3.StatsSecrets(contourConfiguration.Envoy.Metrics.TLS)),
 		&xdscache_v3.RouteCache{},
 		&xdscache_v3.ClusterCache{},

cmd/contour/servecontext.go

@@ -497,6 +497,14 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co
 		Port:    ctx.statsPort,
 	}
 
+	var envoyOMEnforcedHealthListenerConfig *contour_v1alpha1.HealthConfig
+	if ctx.Config.OMEnforcedHealthListener != nil {
+		envoyOMEnforcedHealthListenerConfig = &contour_v1alpha1.HealthConfig{
+			Address: ctx.Config.OMEnforcedHealthListener.Address,
+			Port:    ctx.Config.OMEnforcedHealthListener.Port,
+		}
+	}
+
 	// Override metrics endpoint info from config files
 	//
 	// Note!
@@ -583,6 +591,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co
 				XffNumTrustedHops: &ctx.Config.Network.XffNumTrustedHops,
 				EnvoyAdminPort:    &ctx.Config.Network.EnvoyAdminPort,
 			},
+			OMEnforcedHealth: envoyOMEnforcedHealthListenerConfig,
 		},
 		Gateway: gatewayConfig,
 		HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{

cmd/contour/servecontext_test.go

@@ -487,6 +487,7 @@ func TestConvertServeContext(t *testing.T) {
 					EnvoyAdminPort:    ptr.To(9001),
 					XffNumTrustedHops: ptr.To(uint32(0)),
 				},
+				OMEnforcedHealth: nil,
 			},
 			Gateway: nil,
 			HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{
@@ -902,6 +903,22 @@ func TestConvertServeContext(t *testing.T) {
 				return cfg
 			},
 		},
+		"envoy overload manager health checks": {
+			getServeContext: func(ctx *serveContext) *serveContext {
+				ctx.Config.OMEnforcedHealthListener = &config.OMEnforcedHealthListenerConfig{
+					Address: "0.0.0.0",
+					Port:    8005,
+				}
+				return ctx
+			},
+			getContourConfiguration: func(cfg contour_v1alpha1.ContourConfigurationSpec) contour_v1alpha1.ContourConfigurationSpec {
+				cfg.Envoy.OMEnforcedHealth = &contour_v1alpha1.HealthConfig{
+					Address: "0.0.0.0",
+					Port:    8005,
+				}
+				return cfg
+			},
+		},
 	}
 
 	for name, tc := range cases {

examples/contour/01-crds.yaml

@@ -522,6 +522,20 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -4324,6 +4338,20 @@ spec:
                             format: int32
                             type: integer
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.

examples/render/contour-deployment.yaml

@@ -742,6 +742,20 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -4544,6 +4558,20 @@ spec:
                             format: int32
                             type: integer
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.

examples/render/contour-gateway-provisioner.yaml

@@ -533,6 +533,20 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -4335,6 +4349,20 @@ spec:
                             format: int32
                             type: integer
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.

examples/render/contour-gateway.yaml

@@ -558,6 +558,20 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -4360,6 +4374,20 @@ spec:
                             format: int32
                             type: integer
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.

examples/render/contour.yaml

@@ -742,6 +742,20 @@ spec:
                         format: int32
                         type: integer
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -4544,6 +4558,20 @@ spec:
                             format: int32
                             type: integer
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.