From ac88b996655d5155b1b26d2bc5aa672f59cd366b Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 05:54:43 +0000 Subject: [PATCH 01/47] chore: update devcontainer go version --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4d976f5ab..97e2e0e73 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "build": { "dockerfile": "Dockerfile", "args": { - "VARIANT": "1.22-bullseye" + "VARIANT": "1.23-bullseye" } }, "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], From 6c8c6b64e5aead43223edbe0224b2aae98f860b5 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 05:55:27 +0000 Subject: [PATCH 02/47] chore: refresh toolcain --- hack/toolchain.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hack/toolchain.sh b/hack/toolchain.sh index 0d8138308..9b7048c8d 100755 --- a/hack/toolchain.sh +++ b/hack/toolchain.sh @@ -11,19 +11,19 @@ main() { tools() { go install github.com/google/go-licenses@v1.6.0 - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.57.2 - go install github.com/google/ko@v0.15.2 - go install github.com/mikefarah/yq/v4@v4.43.1 - go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.13.1 - go install sigs.k8s.io/controller-runtime/tools/setup-envtest@0c7827e417acc15f29e7c4bfccede809d372676a - go install sigs.k8s.io/controller-tools/cmd/controller-gen@v0.14.0 - go install github.com/sigstore/cosign/v2/cmd/cosign@v2.2.4 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.61.0 + go install github.com/google/ko@v0.16.0 + go install github.com/mikefarah/yq/v4@v4.44.3 + go install github.com/norwoodj/helm-docs/cmd/helm-docs@v1.14.2 + go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest + go install sigs.k8s.io/controller-tools/cmd/controller-gen@latest + go install github.com/sigstore/cosign/v2/cmd/cosign@v2.4.1 # go install -tags extended github.com/gohugoio/hugo@v0.110.0 - go install golang.org/x/vuln/cmd/govulncheck@v1.0.4 + go install golang.org/x/vuln/cmd/govulncheck@v1.1.3 go install github.com/onsi/ginkgo/v2/ginkgo@latest - go install github.com/rhysd/actionlint/cmd/actionlint@v1.6.27 + go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.3 go install github.com/mattn/goveralls@v0.0.12 - go install github.com/google/go-containerregistry/cmd/crane@v0.19.1 + go install github.com/google/go-containerregistry/cmd/crane@v0.20.2 if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'." From 399191eba6e47f32dff85b9998e6dcf0260cbd41 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:29:33 +0000 Subject: [PATCH 03/47] chore: additional processing on verify (and migration to kube-system) --- .github/actions/e2e/dump-logs/action.yaml | 8 ++-- Makefile | 8 +++- hack/azure/perftest.sh | 11 +++--- .../mutation/conversion_webhooks_injection.sh | 38 +++++++++++++++++++ hack/validation/kubelet.sh | 16 ++++++++ skaffold.yaml | 2 +- 6 files changed, 70 insertions(+), 13 deletions(-) create mode 100755 hack/mutation/conversion_webhooks_injection.sh create mode 100755 hack/validation/kubelet.sh diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml index 573abd247..85c95db1f 100644 --- a/.github/actions/e2e/dump-logs/action.yaml +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -39,18 +39,18 @@ runs: run: | echo "step: controller-logs" AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - POD_NAME=$(kubectl get pods -n karpenter --no-headers -o custom-columns=":metadata.name" | tail -n 1) + POD_NAME=$(kubectl get pods -n kube-system --no-headers -o custom-columns=":metadata.name" | tail -n 1) echo "logs from pod ${POD_NAME}" - kubectl logs "${POD_NAME}" -n karpenter -c controller + kubectl logs "${POD_NAME}" -n kube-system -c controller - name: describe-karpenter-pods shell: bash run: | echo "step: describe-karpenter-pods" AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - kubectl describe pods -n karpenter + kubectl describe pods -n kube-system - name: describe-nodes shell: bash run: | echo "step: describe-nodes" AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - kubectl describe nodes \ No newline at end of file + kubectl describe nodes diff --git a/Makefile b/Makefile index 3a93ed537..24a23ea2c 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ GOFLAGS ?= $(LDFLAGS) WITH_GOFLAGS = GOFLAGS="$(GOFLAGS)" # # CR for local builds of Karpenter -KARPENTER_NAMESPACE ?= karpenter +KARPENTER_NAMESPACE ?= kube-system # Common Directories # TODO: revisit testing tools (temporarily excluded here, for make verify) @@ -80,11 +80,15 @@ verify: toolchain tidy download ## Verify code. Includes dependencies, linting, cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds yq -i '(.spec.versions[0].additionalPrinterColumns[] | select (.name=="Zone")) .jsonPath=".metadata.labels.karpenter\.azure\.com/zone"' \ pkg/apis/crds/karpenter.sh_nodeclaims.yaml + hack/validation/kubelet.sh hack/validation/labels.sh hack/validation/requirements.sh hack/validation/common.sh + cp pkg/apis/crds/* charts/karpenter-crd/templates + hack/mutation/conversion_webhooks_injection.sh hack/github/dependabot.sh - $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) + # TODO: restore linting, excluding code generators (typecheck "main redeclared" issue) + # $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) @git diff --quiet ||\ { echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print " - " $$0}'; \ if [ "${CI}" = true ]; then\ diff --git a/hack/azure/perftest.sh b/hack/azure/perftest.sh index bf0b70bbb..69eda1437 100755 --- a/hack/azure/perftest.sh +++ b/hack/azure/perftest.sh @@ -28,7 +28,7 @@ exec 2>&1 logk="logs/az-perftest-${START}-${replicas}-karpenter.log" # prep -kubectl apply -f hack/azure/general-purpose-small-nodes.yaml +kubectl apply -f hack/azure/general-purpose-small-nodes.yaml kubectl apply -f examples/workloads/inflate.yaml # scale up @@ -37,7 +37,7 @@ kubectl scale --replicas="${replicas}" deployment/inflate time kubectl rollout status deployment/inflate --watch --timeout=2h date ENDUP=$(date ${FMT}) -echo Scale up: ${START} ${ENDUP} ${replicas} +echo Scale up: "${START}" "${ENDUP}" "${replicas}" if [[ "$OSTYPE" == "darwin"* ]]; then # macOS, use BSD date syntax ENDUPKUBECTL=$(date -u +"%Y-%m-%dT%H:%M:%SZ") @@ -45,7 +45,7 @@ else # Linux, use GNU date syntax ENDUPKUBECTL=$(date --iso-8601=seconds) fi -kubectl logs deployment/karpenter -n karpenter --since-time="${STARTKUBECTL}" > "${logk}" +kubectl logs deployment/karpenter -n kube-system --since-time="${STARTKUBECTL}" > "${logk}" # scale down sleep 30 @@ -53,10 +53,9 @@ kubectl scale --replicas=0 deployment/inflate date kubectl delete --wait=false nodes -l karpenter.sh/nodepool time kubectl wait --for=delete nodes -l karpenter.sh/nodepool --timeout=2h -ENDDOWN=$(date ${FMT}) +# ENDDOWN=$(date ${FMT}) date # review -kubectl logs deployment/karpenter -n karpenter --since-time="${ENDUPKUBECTL}" >> "${logk}" +kubectl logs deployment/karpenter -n kube-system --since-time="${ENDUPKUBECTL}" >> "${logk}" az resource list -o table --tag=karpenter.sh_nodepool=sm-general-purpose - diff --git a/hack/mutation/conversion_webhooks_injection.sh b/hack/mutation/conversion_webhooks_injection.sh new file mode 100755 index 000000000..a25bd061a --- /dev/null +++ b/hack/mutation/conversion_webhooks_injection.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Add the conversion stanza to the CRD spec to enable conversion via webhook +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml +yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + +# Update to the karpenter-crd charts + +# Add the conversion stanza to the CRD spec to enable conversion via webhook +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml diff --git a/hack/validation/kubelet.sh b/hack/validation/kubelet.sh new file mode 100755 index 000000000..a29b44498 --- /dev/null +++ b/hack/validation/kubelet.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -euo pipefail + +# Kubelet Validation + +# The regular expression adds validation for kubelet.kubeReserved and kubelet.systemReserved values of the map are resource.Quantity +# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 +# AKSNodeClass Validation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.kubeReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.systemReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml + +# The regular expression is a validation for kubelet.evictionHard and kubelet.evictionSoft are percentage or a resource.Quantity +# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 +# AKSNodeClass Validation: +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionHard.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionSoft.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml diff --git a/skaffold.yaml b/skaffold.yaml index fdbe479c7..e62f28288 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -16,7 +16,7 @@ manifests: - name: karpenter chartPath: charts/karpenter skipBuildDependencies: true - namespace: karpenter + namespace: kube-system createNamespace: true setValueTemplates: controller.image.repository: "{{.IMAGE_REPO_controller}}" From ca59c0b8dafc1423a5e0ded4fef296dbc016a60a Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:30:12 +0000 Subject: [PATCH 04/47] chore: bump dependencies --- go.mod | 69 ++++++++++++++-------------- go.sum | 140 +++++++++++++++++++++++++++------------------------------ 2 files changed, 100 insertions(+), 109 deletions(-) diff --git a/go.mod b/go.mod index fb5bea01e..af421005d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/Azure/karpenter-provider-azure -go 1.22 +go 1.23.0 require ( github.com/Azure/azure-kusto-go v0.16.1 @@ -18,6 +18,7 @@ require ( github.com/Azure/skewer v0.0.19 github.com/Pallinder/go-randomdata v1.2.0 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 + github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 github.com/blang/semver/v4 v4.0.0 github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 @@ -34,16 +35,16 @@ require ( go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 golang.org/x/crypto v0.27.0 - k8s.io/api v0.29.3 - k8s.io/apiextensions-apiserver v0.29.3 - k8s.io/apimachinery v0.29.3 - k8s.io/client-go v0.29.3 - k8s.io/klog/v2 v2.120.1 - k8s.io/utils v0.0.0-20240102154912-e7106e64919e - knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd + k8s.io/api v0.30.3 + k8s.io/apiextensions-apiserver v0.30.3 + k8s.io/apimachinery v0.30.3 + k8s.io/client-go v0.30.3 + k8s.io/klog/v2 v2.130.1 + k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 + knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7 sigs.k8s.io/cloud-provider-azure v1.29.3 - sigs.k8s.io/controller-runtime v0.17.3 - sigs.k8s.io/karpenter v0.36.1 + sigs.k8s.io/controller-runtime v0.18.4 + sigs.k8s.io/karpenter v1.0.4 ) require ( @@ -72,21 +73,20 @@ require ( github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/evanphx/json-patch/v5 v5.8.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect - github.com/gobuffalo/flect v1.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect @@ -97,7 +97,7 @@ require ( github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -122,37 +122,36 @@ require ( github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect - go.opentelemetry.io/otel v1.21.0 // indirect - go.opentelemetry.io/otel/metric v1.21.0 // indirect - go.opentelemetry.io/otel/trace v1.21.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect go.uber.org/mock v0.4.0 // indirect - golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.21.0 // indirect + golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect + golang.org/x/net v0.29.0 // indirect + golang.org/x/oauth2 v0.22.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.6.0 // indirect golang.org/x/tools v0.24.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect - google.golang.org/api v0.146.0 // indirect - google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect - google.golang.org/grpc v1.59.0 // indirect + google.golang.org/api v0.183.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a // indirect + google.golang.org/grpc v1.65.0 // indirect google.golang.org/protobuf v1.34.2 // indirect gopkg.in/dnaeon/go-vcr.v3 v3.2.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.29.3 // indirect - k8s.io/component-base v0.29.3 // indirect - k8s.io/component-helpers v0.29.3 // indirect - k8s.io/csi-translation-lib v0.29.3 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/cloud-provider v0.30.3 // indirect + k8s.io/component-base v0.30.3 // indirect + k8s.io/component-helpers v0.30.3 // indirect + k8s.io/csi-translation-lib v0.30.3 // indirect + k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26 // indirect sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/go.sum b/go.sum index f00c53a27..36bbeff0f 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,8 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306 h1:0dzaVod1XLEc38H4IB+KOgStoCt8RkCVI4t+XsSPrWE= +github.com/awslabs/operatorpkg v0.0.0-20240805231134-67d0acfb6306/go.mod h1:7u2ugtOiWSvqqwNlnQ8W+2TjwnSTbHoMHnR1AKpKVMA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -140,20 +142,19 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.8.0 h1:lRj6N9Nci7MvzrXuX6HFzU8XjmhPiXPlsKEy1u0KQro= -github.com/evanphx/json-patch/v5 v5.8.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= @@ -183,14 +184,12 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= @@ -202,8 +201,6 @@ github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaC github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= -github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= -github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -213,8 +210,6 @@ github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -284,8 +279,8 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= @@ -321,7 +316,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -444,14 +438,14 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= @@ -487,8 +481,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa h1:ELnwvuAXPNtPk1TJRuGkI9fDTwym6AYBu0qzT8AcHdI= +golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa/go.mod h1:akd2r19cwCdwSwWeIdzYQGa/EZZyqcOdwWiwj5L5eKQ= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -552,8 +546,8 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= +golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -561,8 +555,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs= -golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= +golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -648,8 +642,8 @@ golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= +golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -719,8 +713,8 @@ google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.146.0 h1:9aBYT4vQXt9dhCuLNfwfd3zpwu8atg0yPkjBymwSrOM= -google.golang.org/api v0.146.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= +google.golang.org/api v0.183.0 h1:PNMeRDwo1pJdgNcFQ9GstuLe/noWKIc89pRWRLMvLwE= +google.golang.org/api v0.183.0/go.mod h1:q43adC5/pHoSZTx5h2mSmdF7NcyfW9JuDyIOJAgS9ZQ= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -758,12 +752,10 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c h1:ml3TAUoIIzQUtX88s/icpXCFW9lV5VwsuIuS1htNjKY= -google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:MugzuwC+GYOxyF0XUGQvsT97bOgWCV7MM1XMc5FZv8E= -google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c h1:0RtEmmHjemvUXloH7+RuBSIw7n+GEHMOMY1CkGYnWq4= -google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:Wth13BrWMRN/G+guBLupKa6fslcWZv14R0ZKDRkNfY8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a h1:KyUe15n7B1YCu+kMmPtlXxgkLQbp+Dw0tCRZf9Sd+CE= +google.golang.org/genproto/googleapis/api v0.0.0-20240808171019-573a1156607a/go.mod h1:4+X6GvPs+25wZKbQq9qyAXrwIRExv7w0Ea6MgZLZiDM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a h1:EKiZZXueP9/T68B8Nl0GAx9cjbQnCId0yP3qPMgaaHs= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240808171019-573a1156607a/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -777,8 +769,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -825,30 +817,30 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= -k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= -k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= -k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= -k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= -k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= -k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= -k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= -k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= -k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= -k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= -k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= -k8s.io/component-helpers v0.29.3 h1:1dqZswuZgT2ZMixYeORyCUOAApXxgsvjVSgfoUT+P4o= -k8s.io/component-helpers v0.29.3/go.mod h1:yiDqbRQrnQY+sPju/bL7EkwDJb6LVOots53uZNMZBos= -k8s.io/csi-translation-lib v0.29.3 h1:GNYCE0f86K3Xkyrk7WKKwQZkJrum6QQapbOzYxZv6Mg= -k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= -k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= -k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= -k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= -knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= +k8s.io/api v0.30.3 h1:ImHwK9DCsPA9uoU3rVh4QHAHHK5dTSv1nxJUapx8hoQ= +k8s.io/api v0.30.3/go.mod h1:GPc8jlzoe5JG3pb0KJCSLX5oAFIW3/qNJITlDj8BH04= +k8s.io/apiextensions-apiserver v0.30.3 h1:oChu5li2vsZHx2IvnGP3ah8Nj3KyqG3kRSaKmijhB9U= +k8s.io/apiextensions-apiserver v0.30.3/go.mod h1:uhXxYDkMAvl6CJw4lrDN4CPbONkF3+XL9cacCT44kV4= +k8s.io/apimachinery v0.30.3 h1:q1laaWCmrszyQuSQCfNB8cFgCuDAoPszKY4ucAjDwHc= +k8s.io/apimachinery v0.30.3/go.mod h1:iexa2somDaxdnj7bha06bhb43Zpa6eWH8N8dbqVjTUc= +k8s.io/client-go v0.30.3 h1:bHrJu3xQZNXIi8/MoxYtZBBWQQXwy16zqJwloXXfD3k= +k8s.io/client-go v0.30.3/go.mod h1:8d4pf8vYu665/kUbsxWAQ/JDBNWqfFeZnvFiVdmx89U= +k8s.io/cloud-provider v0.30.3 h1:SNWZmllTymOTzIPJuhtZH6il/qVi75dQARRQAm9k6VY= +k8s.io/cloud-provider v0.30.3/go.mod h1:Ax0AVdHnM7tMYnJH1Ycy4SMBD98+4zA+tboUR9eYsY8= +k8s.io/component-base v0.30.3 h1:Ci0UqKWf4oiwy8hr1+E3dsnliKnkMLZMVbWzeorlk7s= +k8s.io/component-base v0.30.3/go.mod h1:C1SshT3rGPCuNtBs14RmVD2xW0EhRSeLvBh7AGk1quA= +k8s.io/component-helpers v0.30.3 h1:KPc8l0eGx9Wg2OcKc58k9ozNcVcOInAi3NGiuS2xJ/c= +k8s.io/component-helpers v0.30.3/go.mod h1:VOQ7g3q+YbKWwKeACG2BwPv4ftaN8jXYJ5U3xpzuYAE= +k8s.io/csi-translation-lib v0.30.3 h1:wBaPWnOi14/vANRIrp8pmbdx/Pgz2QRcroH7wkodezc= +k8s.io/csi-translation-lib v0.30.3/go.mod h1:3AizNZbDttVDH1RO0x1yGEQP74e9Xbfb60IBP1oWO1o= +k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= +k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8 h1:1Wof1cGQgA5pqgo8MxKPtf+qN6Sh/0JzznmeGPm1HnE= +k8s.io/kube-openapi v0.0.0-20240808142205-8e686545bdb8/go.mod h1:Os6V6dZwLNii3vxFpxcNaTmH8LJJBkOTg1N0tOA0fvA= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= +k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7 h1:ClGJ7Q7iQC9qxov1lNys7xC0MqDGaw/ysGEuyqdIwAM= +knative.dev/pkg v0.0.0-20240910170930-fdbc0b5adde7/go.mod h1:fL50zroI/eGDwkkWTJsOpto46G8T/xmTjBzkwnh/S48= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= @@ -858,12 +850,12 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26 h1:BHauRhfjzs4UWu/yiLw82WK sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.26/go.mod h1:02JRJ7ioAoT9PZzIxlR4Kw7WbejsMIy1eeDyYX8sgvk= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1 h1:Lp0nALZmvMJoiVeVV6XjnZv1uClfArnThhuDAjaqE5A= sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.1/go.mod h1:pPkJPx/eMVWP3R+LhPoOYGoY7lywcMJev5L2uSfH+Jo= -sigs.k8s.io/controller-runtime v0.17.3 h1:65QmN7r3FWgTxDMz9fvGnO1kbf2nu+acg9p2R9oYYYk= -sigs.k8s.io/controller-runtime v0.17.3/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= +sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHvm5BZw= +sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.36.1 h1:NP36T+uTTeO9fAiWttEBhXrtCiDPJkiyLNZWjFSIkb0= -sigs.k8s.io/karpenter v0.36.1/go.mod h1:fieFojxOec/l0tDmFT7R+g/Y+SGQbL9VlcYO8xb3sLo= +sigs.k8s.io/karpenter v1.0.4 h1:tE4gbYjXZ2uny35QmrKFSuMhxkIfGCqCulXhwyN2D3c= +sigs.k8s.io/karpenter v1.0.4/go.mod h1:3NLmsnHHw8p4VutpjTOPUZyhE3qH6yGTs8O94Lsu8uw= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= From 5f9e04c3761ba6d8e0aaf1b21e2fa1fc11940b26 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:31:15 +0000 Subject: [PATCH 05/47] chore: refresh Helm charts --- .../karpenter.azure.com_aksnodeclasses.yaml | 250 +++- .../templates/karpenter.sh_nodeclaims.yaml | 830 ++++++++++++- .../templates/karpenter.sh_nodepools.yaml | 1081 ++++++++++++++++- charts/karpenter-crd/values.yaml | 7 + charts/karpenter/crds | 1 - charts/karpenter/templates/_helpers.tpl | 43 +- .../karpenter/templates/clusterrole-core.yaml | 26 +- charts/karpenter/templates/clusterrole.yaml | 12 +- .../templates/configmap-logging.yaml | 41 - charts/karpenter/templates/deployment.yaml | 58 +- charts/karpenter/templates/role.yaml | 20 +- .../templates/secret-webhook-cert.yaml | 2 + charts/karpenter/templates/service.yaml | 6 + charts/karpenter/templates/webhooks-core.yaml | 69 -- charts/karpenter/values.yaml | 97 +- 15 files changed, 2272 insertions(+), 271 deletions(-) mode change 120000 => 100644 charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml mode change 120000 => 100644 charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml mode change 120000 => 100644 charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml create mode 100644 charts/karpenter-crd/values.yaml delete mode 120000 charts/karpenter/crds delete mode 100644 charts/karpenter/templates/configmap-logging.yaml delete mode 100644 charts/karpenter/templates/webhooks-core.yaml diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml deleted file mode 120000 index 4917e92a9..000000000 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml new file mode 100644 index 000000000..9b5afdeb9 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -0,0 +1,249 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: aksnodeclasses.karpenter.azure.com +spec: + group: karpenter.azure.com + names: + categories: + - karpenter + kind: AKSNodeClass + listKind: AKSNodeClassList + plural: aksnodeclasses + shortNames: + - aksnc + - aksncs + singular: aksnodeclass + scope: Cluster + versions: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: AKSNodeClass is the Schema for the AKSNodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. + This will contain configuration necessary to launch instances in AKS. + properties: + imageFamily: + default: Ubuntu2204 + description: ImageFamily is the image family that instances use. + enum: + - Ubuntu2204 + - AzureLinux + type: string + imageVersion: + description: ImageVersion is the image version that instances use. + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + osDiskSizeGB: + default: 128 + description: osDiskSizeGB is the size of the OS disk in GB. + format: int32 + minimum: 100 + type: integer + tags: + additionalProperties: + type: string + description: Tags to be applied on Azure resources like instances. + type: object + type: object + status: + description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml deleted file mode 120000 index 3f572b575..000000000 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml new file mode 100644 index 000000000..269ca1387 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -0,0 +1,829 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodeclaims.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodeClaim + listKind: NodeClaimList + plural: nodeclaims + singular: nodeclaim + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string + - jsonPath: .metadata.labels.karpenter\.azure\.com/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.azure.com" is restricted + rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml deleted file mode 120000 index 36d2d1dd9..000000000 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ /dev/null @@ -1 +0,0 @@ -../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml new file mode 100644 index 000000000..f0330c96b --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -0,0 +1,1080 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodepools.karpenter.sh +spec: + group: karpenter.sh + names: + categories: + - karpenter + kind: NodePool + listKind: NodePoolList + plural: nodepools + singular: nodepool + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.azure.com" is restricted + rule: self.all(x, x in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.azure.com" is restricted + rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenUnderutilized" if not specified + enum: + - WhenEmpty + - WhenUnderutilized + type: string + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + type: object + x-kubernetes-validations: + - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized + rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' + - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty + rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + apiVersion: + description: API version of the referent + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + maxProperties: 0 + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} +{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} diff --git a/charts/karpenter-crd/values.yaml b/charts/karpenter-crd/values.yaml new file mode 100644 index 000000000..feee346e8 --- /dev/null +++ b/charts/karpenter-crd/values.yaml @@ -0,0 +1,7 @@ +webhook: + # -- Whether to enable the webhooks. + enabled: true + serviceName: karpenter + serviceNamespace: "" + # -- The container port to use for the webhook. + port: 8443 diff --git a/charts/karpenter/crds b/charts/karpenter/crds deleted file mode 120000 index e33998437..000000000 --- a/charts/karpenter/crds +++ /dev/null @@ -1 +0,0 @@ -../../pkg/apis/crds \ No newline at end of file diff --git a/charts/karpenter/templates/_helpers.tpl b/charts/karpenter/templates/_helpers.tpl index 90415751c..701fab128 100644 --- a/charts/karpenter/templates/_helpers.tpl +++ b/charts/karpenter/templates/_helpers.tpl @@ -141,44 +141,12 @@ This works because Helm treats dictionaries as mutable objects and allows passin {{- end }} {{- end }} -{{/* -Flatten Settings Map using "." syntax -*/}} -{{- define "flattenSettings" -}} -{{- $map := first . -}} -{{- $label := last . -}} -{{- range $key := (keys $map | uniq | sortAlpha) }} - {{- $sublabel := $key -}} - {{- $val := (get $map $key) -}} - {{- if $label -}} - {{- $sublabel = list $label $key | join "." -}} - {{- end -}} - {{/* Special-case "tags" since we want this to be a JSON object */}} - {{- if eq $key "tags" -}} - {{- if not (kindIs "invalid" $val) -}} - {{- $sublabel | quote | nindent 2 }}: {{ $val | toJson | quote }} - {{- end -}} - {{- else if kindOf $val | eq "map" -}} - {{- list $val $sublabel | include "flattenSettings" -}} - {{- else -}} - {{- if not (kindIs "invalid" $val) -}} - {{- $sublabel | quote | nindent 2 -}}: {{ $val | quote }} - {{- end -}} -{{- end -}} -{{- end -}} -{{- end -}} - {{/* Flatten the stdout logging outputs from args provided */}} {{- define "karpenter.outputPathsList" -}} {{ $paths := list -}} -{{- range .Values.controller.outputPaths -}} - {{- if not (has (printf "%s" . | quote) $paths) -}} - {{- $paths = printf "%s" . | quote | append $paths -}} - {{- end -}} -{{- end -}} -{{- range .Values.logConfig.outputPaths -}} +{{- range .Values.logOutputPaths -}} {{- if not (has (printf "%s" . | quote) $paths) -}} {{- $paths = printf "%s" . | quote | append $paths -}} {{- end -}} @@ -191,15 +159,10 @@ Flatten the stderr logging outputs from args provided */}} {{- define "karpenter.errorOutputPathsList" -}} {{ $paths := list -}} -{{- range .Values.controller.errorOutputPaths -}} - {{- if not (has (printf "%s" . | quote) $paths) -}} - {{- $paths = printf "%s" . | quote | append $paths -}} - {{- end -}} -{{- end -}} -{{- range .Values.logConfig.errorOutputPaths -}} +{{- range .Values.logErrorOutputPaths -}} {{- if not (has (printf "%s" . | quote) $paths) -}} {{- $paths = printf "%s" . | quote | append $paths -}} {{- end -}} {{- end -}} {{ $paths | join ", " }} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index 5210e2b59..d46de7d70 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -36,16 +36,16 @@ rules: resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] verbs: ["get", "list", "watch"] - apiGroups: ["storage.k8s.io"] - resources: ["storageclasses", "csinodes"] + resources: ["storageclasses", "csinodes", "volumeattachments"] verbs: ["get", "watch", "list"] - apiGroups: ["apps"] resources: ["daemonsets", "deployments", "replicasets", "statefulsets"] verbs: ["list", "watch"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations", "mutatingwebhookconfigurations"] - verbs: ["get", "watch", "list"] -{{- end }} + {{- if .Values.webhook.enabled }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["watch", "list"] + {{- end }} - apiGroups: ["policy"] resources: ["poddisruptionbudgets"] verbs: ["get", "list", "watch"] @@ -61,16 +61,18 @@ rules: verbs: ["create", "patch"] - apiGroups: [""] resources: ["nodes"] - verbs: ["create", "patch", "delete"] + verbs: ["patch", "delete", "update"] - apiGroups: [""] resources: ["pods/eviction"] verbs: ["create"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["delete"] + {{- if .Values.webhook.enabled }} + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.sh", "validation.webhook.config.karpenter.sh"] -{{- end }} + {{- end }} {{- with .Values.additionalClusterRoleRules -}} {{ toYaml . | nindent 2 }} {{- end -}} diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index b981c1bc5..729a0cbb8 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -33,16 +33,6 @@ rules: resources: ["aksnodeclasses"] verbs: ["get", "list", "watch"] # Write - - apiGroups: ["karpenter.kubernetes.com"] + - apiGroups: ["karpenter.azure.com"] resources: ["aksnodeclasses", "aksnodeclasses/status"] verbs: ["patch", "update"] -{{- if .Values.webhook.enabled }} - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["validatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["validation.webhook.karpenter.azure.com"] - - apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["update"] - resourceNames: ["defaulting.webhook.karpenter.azure.com"] -{{- end }} diff --git a/charts/karpenter/templates/configmap-logging.yaml b/charts/karpenter/templates/configmap-logging.yaml deleted file mode 100644 index 0de88c183..000000000 --- a/charts/karpenter/templates/configmap-logging.yaml +++ /dev/null @@ -1,41 +0,0 @@ -{{- if .Values.logConfig.enabled }} -apiVersion: v1 -kind: ConfigMap -metadata: - name: config-logging - namespace: {{ .Release.Namespace }} - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -data: - # https://github.com/uber-go/zap/blob/aa3e73ec0896f8b066ddf668597a02f89628ee50/config.go - zap-logger-config: | - { - "level": "{{ .Values.logConfig.logLevel.global }}", - "development": false, - "disableStacktrace": true, - "disableCaller": true, - "sampling": { - "initial": 100, - "thereafter": 100 - }, - "outputPaths": [{{ include "karpenter.outputPathsList" . }}], - "errorOutputPaths": [{{ include "karpenter.errorOutputPathsList" . }}], - "encoding": "{{ or .Values.logEncoding .Values.logConfig.logEncoding }}", - "encoderConfig": { - "timeKey": "time", - "levelKey": "level", - "nameKey": "logger", - "callerKey": "caller", - "messageKey": "message", - "stacktraceKey": "stacktrace", - "levelEncoder": "capital", - "timeEncoder": "iso8601" - } - } - loglevel.controller: {{ or .Values.controller.logLevel .Values.logConfig.logLevel.controller }} - loglevel.webhook: {{ or .Values.webhook.logLevel .Values.logConfig.logLevel.webhook }} -{{- end }} diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index 066497fbc..5b258ff4f 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -36,6 +36,10 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} serviceAccountName: {{ include "karpenter.serviceAccountName" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} {{- with .Values.priorityClassName }} priorityClassName: {{ . | quote }} {{- end }} @@ -55,8 +59,8 @@ spec: containers: - name: controller securityContext: - runAsUser: 65536 - runAsGroup: 65536 + runAsUser: 65532 + runAsGroup: 65532 runAsNonRoot: true seccompProfile: type: RuntimeDefault @@ -77,10 +81,9 @@ spec: value: "{{ .Values.webhook.port }}" - name: WEBHOOK_METRICS_PORT value: "{{ .Values.webhook.metrics.port }}" - {{- else }} - - name: DISABLE_WEBHOOK - value: "true" {{- end }} + - name: DISABLE_WEBHOOK + value: "{{ not .Values.webhook.enabled }}" {{- with .Values.logLevel }} - name: LOG_LEVEL value: "{{ . }}" @@ -99,6 +102,32 @@ spec: containerName: controller divisor: "0" resource: limits.memory + - name: FEATURE_GATES + value: "SpotToSpotConsolidation={{ .Values.settings.featureGates.spotToSpotConsolidation }}" + {{- with .Values.settings.batchMaxDuration }} + - name: BATCH_MAX_DURATION + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.batchIdleDuration }} + - name: BATCH_IDLE_DURATION + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.clusterCABundle }} + - name: CLUSTER_CA_BUNDLE + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.clusterName }} + - name: CLUSTER_NAME + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.clusterEndpoint }} + - name: CLUSTER_ENDPOINT + value: "{{ . }}" + {{- end }} + {{- with .Values.settings.vmMemoryOverheadPercent }} + - name: VM_MEMORY_OVERHEAD_PERCENT + value: "{{ . }}" + {{- end }} {{- with .Values.controller.env }} {{- toYaml . | nindent 12 }} {{- end }} @@ -114,13 +143,13 @@ spec: - name: webhook-metrics containerPort: {{ .Values.webhook.metrics.port }} protocol: TCP + - name: https-webhook + containerPort: {{ .Values.webhook.port }} + protocol: TCP {{- end }} - name: http containerPort: {{ .Values.controller.healthProbe.port }} protocol: TCP - - name: https-webhook - containerPort: {{ .Values.webhook.port }} - protocol: TCP livenessProbe: initialDelaySeconds: 30 timeoutSeconds: 30 @@ -137,12 +166,8 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.controller.extraVolumeMounts) }} + {{- if .Values.controller.extraVolumeMounts }} volumeMounts: - {{- if .Values.logConfig.enabled }} - - name: config-logging - mountPath: /etc/karpenter/logging - {{- end }} {{- with .Values.controller.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -179,13 +204,8 @@ spec: tolerations: {{- toYaml . | nindent 8 }} {{- end }} - {{- if or (.Values.logConfig.enabled) (.Values.extraVolumes) }} + {{- if .Values.extraVolumes }} volumes: - {{- if .Values.logConfig.enabled }} - - name: config-logging - configMap: - name: config-logging - {{- end }} {{- with .Values.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} diff --git a/charts/karpenter/templates/role.yaml b/charts/karpenter/templates/role.yaml index c7011ffe1..ad83a6f14 100644 --- a/charts/karpenter/templates/role.yaml +++ b/charts/karpenter/templates/role.yaml @@ -14,36 +14,29 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["get", "watch"] +{{- if .Values.webhook.enabled }} - apiGroups: [""] - resources: ["configmaps", "namespaces", "secrets"] + resources: ["configmaps", "secrets"] verbs: ["get", "list", "watch"] +{{- end }} # Write +{{- if .Values.webhook.enabled }} - apiGroups: [""] resources: ["secrets"] verbs: ["update"] - resourceNames: ["{{ include "karpenter.fullname" . }}-cert"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["update", "patch", "delete"] resourceNames: - - config-logging + - "{{ include "karpenter.fullname" . }}-cert" +{{- end }} - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["patch", "update"] resourceNames: - "karpenter-leader-election" - - "webhook.configmapwebhook.00-of-01" - - "webhook.defaultingwebhook.00-of-01" - - "webhook.validationwebhook.00-of-01" - - "webhook.webhookcertificates.00-of-01" # Cannot specify resourceNames on create # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["create"] - - apiGroups: [""] - resources: ["configmaps"] - verbs: ["create"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role @@ -83,4 +76,3 @@ rules: - apiGroups: ["coordination.k8s.io"] resources: ["leases"] verbs: ["delete"] - diff --git a/charts/karpenter/templates/secret-webhook-cert.yaml b/charts/karpenter/templates/secret-webhook-cert.yaml index a364f9b71..b17309f57 100644 --- a/charts/karpenter/templates/secret-webhook-cert.yaml +++ b/charts/karpenter/templates/secret-webhook-cert.yaml @@ -1,3 +1,4 @@ +{{- if .Values.webhook.enabled }} apiVersion: v1 kind: Secret metadata: @@ -10,3 +11,4 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} # data: {} # Injected by karpenter-webhook +{{- end }} diff --git a/charts/karpenter/templates/service.yaml b/charts/karpenter/templates/service.yaml index 5236489ae..2c880b708 100644 --- a/charts/karpenter/templates/service.yaml +++ b/charts/karpenter/templates/service.yaml @@ -16,9 +16,15 @@ spec: port: {{ .Values.controller.metrics.port }} targetPort: http-metrics protocol: TCP + {{- if .Values.webhook.enabled }} + - name: webhook-metrics + port: {{ .Values.webhook.metrics.port }} + targetPort: webhook-metrics + protocol: TCP - name: https-webhook port: {{ .Values.webhook.port }} targetPort: https-webhook protocol: TCP + {{- end }} selector: {{- include "karpenter.selectorLabels" . | nindent 4 }} diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml deleted file mode 100644 index 004167011..000000000 --- a/charts/karpenter/templates/webhooks-core.yaml +++ /dev/null @@ -1,69 +0,0 @@ -{{- if .Values.webhook.enabled }} -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - rules: - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodeclaims - - nodeclaims/status - scope: '*' - - apiGroups: - - karpenter.sh - apiVersions: - - v1beta1 - operations: - - CREATE - - UPDATE - resources: - - nodepools - - nodepools/status - scope: '*' ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: validation.webhook.config.karpenter.sh - labels: - {{- include "karpenter.labels" . | nindent 4 }} - {{- with .Values.additionalAnnotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -webhooks: - - name: validation.webhook.config.karpenter.sh - admissionReviewVersions: ["v1"] - clientConfig: - service: - name: {{ include "karpenter.fullname" . }} - namespace: {{ .Release.Namespace }} - port: {{ .Values.webhook.port }} - failurePolicy: Fail - sideEffects: None - objectSelector: - matchLabels: - app.kubernetes.io/part-of: {{ template "karpenter.name" . }} -{{- end }} diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 1f056dd2b..4c4260d46 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -27,7 +27,9 @@ serviceMonitor: enabled: false # -- Additional labels for the ServiceMonitor. additionalLabels: {} - # -- Endpoint configuration for the ServiceMonitor. + # -- Configuration on `http-metrics` endpoint for the ServiceMonitor. + # Not to be used to add additional endpoints. + # See the Prometheus operator documentation for configurable fields https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#endpoint endpointConfig: {} # -- Number of replicas. replicas: 2 @@ -44,6 +46,9 @@ podAnnotations: {} podDisruptionBudget: name: karpenter maxUnavailable: 1 +# -- SecurityContext for the pod. +podSecurityContext: + fsGroup: 65532 # -- PriorityClass name for the pod. priorityClassName: system-cluster-critical # -- Override the default termination grace period for the pod. @@ -52,7 +57,7 @@ terminationGracePeriodSeconds: # This is required when using a custom CNI. hostNetwork: false # -- Configure the DNS Policy for the pod -dnsPolicy: Default +dnsPolicy: ClusterFirst # -- Configure DNS Config for the pod dnsConfig: {} # options: @@ -76,31 +81,23 @@ affinity: topologySpreadConstraints: - maxSkew: 1 topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway + whenUnsatisfiable: DoNotSchedule # -- Tolerations to allow the pod to be scheduled to nodes with taints. tolerations: - key: CriticalAddonsOnly operator: Exists # -- Additional volumes for the pod. extraVolumes: [] -# - name: -# projected: -# defaultMode: 420 -# sources: -# - serviceAccountToken: -# audience: -# expirationSeconds: 86400 -# path: token + controller: image: # -- Repository path to the controller image. repository: mcr.microsoft.com/aks/karpenter/controller # -- Tag of the controller image. - tag: v0.32.1 + tag: # -- SHA256 digest of the controller image. # digest: - - digest: sha256:e35e315face303a784e137c5af5683f4421fa8537f1ea70cacdc1673f7f08b28 + digest: sha256:dd095cdcf857c3812f2084a7b20294932f461b0bff912acf58d592faa032fbef # -- Additional environment variables for the controller pod. env: [] # - name: @@ -130,40 +127,26 @@ controller: sidecarVolumeMounts: [] metrics: # -- The container port to use for metrics. - port: 8000 + port: 8080 healthProbe: # -- The container port to use for http health probe. port: 8081 webhook: # -- Whether to enable the webhooks and webhook permissions. - enabled: false + enabled: true # -- The container port to use for the webhook. port: 8443 metrics: # -- The container port to use for webhook metrics. port: 8001 -# -- Global log level -logLevel: debug -# -- Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) -logConfig: - # -- Whether to enable provisioning and mounting the log ConfigMap - enabled: true - # -- Log outputPaths - defaults to stdout only - outputPaths: - - stdout - # -- Log errorOutputPaths - defaults to stderr only - errorOutputPaths: - - stderr - # -- Log encoding - defaults to json - must be one of 'json', 'console' - logEncoding: json - # -- Component-based log configuration - logLevel: - # -- Global log level, defaults to 'debug' - global: debug - # -- Controller log level, defaults to 'debug' - controller: debug - # -- Error log level, defaults to 'error' - webhook: error +# -- Global log level, defaults to 'info' +logLevel: info +# -- Log outputPaths - defaults to stdout only +logOutputPaths: + - stdout +# -- Log errorOutputPaths - defaults to stderr only +logErrorOutputPaths: + - stderr # -- Global Settings to configure Karpenter settings: # -- The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one @@ -173,29 +156,21 @@ settings: # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods # will be batched separately. batchIdleDuration: 1s - # -- Azure-specific configuration values - azure: - # -- Cluster name. - clusterName: "" - # -- Cluster endpoint. - clusterEndpoint: "" - # -- Kubelet client TLS bootstrap token. - kubeletClientTLSBootstrapToken: "" - # -- SSH public key. - sshPublicKey: "" - # -- Network plugin. - networkPlugin: "azure" - # -- Network policy. - networkPolicy: "" - # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types - vmMemoryOverheadPercent: 0.075 - # -- The global tags to use on all Azure infrastructure resources (VMs, etc.) - # TODO: not propagated yet ... - tags: + # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. + clusterCABundle: "" + # -- Cluster name. + clusterName: "" + # -- Cluster endpoint. + clusterEndpoint: "" + # -- The VM memory overhead as a percent that will be subtracted from the total memory for all instance types + vmMemoryOverheadPercent: 0.075 + # -- The global tags to use on all Azure infrastructure resources (VMs, etc.) + # TODO: not propagated yet ... + tags: + # -- Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates # in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features featureGates: - # -- drift is in ALPHA and is disabled by default. - # Setting drift to true enables the drift disruption method to watch for drift between currently deployed nodes - # and the desired state of nodes set in provisioners and node templates - drift: true + # -- spotToSpotConsolidation is ALPHA and is disabled by default. + # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. + spotToSpotConsolidation: false From 92a3e29c576ee02e72b90f680231b8f3112c051e Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:31:54 +0000 Subject: [PATCH 06/47] chore: update golangci config --- .golangci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.golangci.yaml b/.golangci.yaml index c5731b8b1..b26e479f9 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -10,7 +10,7 @@ linters: - bidichk - errorlint - errcheck - - exportloopref + - copyloopvar - gosec - revive - stylecheck From ae007bd86c30e0fb9b7ba4b1c1707e64ec48b795 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:32:36 +0000 Subject: [PATCH 07/47] chore: remove feature gate for drift --- karpenter-values-template.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/karpenter-values-template.yaml b/karpenter-values-template.yaml index 5db515e03..5e3815b30 100644 --- a/karpenter-values-template.yaml +++ b/karpenter-values-template.yaml @@ -2,8 +2,6 @@ replicas: 1 # for better debugging experience controller: env: - - name: FEATURE_GATES - value: Drift=true - name: LEADER_ELECT # disable leader election for better debugging / troubleshooting experience value: "false" # disable HTTP/2 to reduce ARM throttling on large-scale tests; From 40188101ee8528361e731f1613b7e772a6ea5943 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:40:07 +0000 Subject: [PATCH 08/47] chore: update pre-commit tooling --- .pre-commit-config.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 03748e909..389f278ad 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: - repo: https://github.com/gitleaks/gitleaks - rev: v8.18.1 + rev: v8.20.1 hooks: - id: gitleaks - repo: https://github.com/golangci/golangci-lint - rev: v1.55.2 + rev: v1.61.0 hooks: - id: golangci-lint - repo: https://github.com/jumanjihouse/pre-commit-hooks @@ -12,11 +12,11 @@ repos: hooks: - id: shellcheck - repo: https://github.com/crate-ci/typos - rev: v1.17.2 + rev: v1.26.0 hooks: - id: typos - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v5.0.0 hooks: - id: end-of-file-fixer - id: trailing-whitespace From 6031c417cdad48fa292d3a71e38102fe473bdfd6 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:48:08 +0000 Subject: [PATCH 09/47] chore: update the shape of main --- cmd/controller/main.go | 9 +++++---- cmd/controller/main_ccp.go | 8 +++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index def58c202..3977d695f 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -22,12 +22,11 @@ import ( "github.com/samber/lo" "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" + "github.com/Azure/karpenter-provider-azure/pkg/controllers" "github.com/Azure/karpenter-provider-azure/pkg/operator" - controllers "github.com/Azure/karpenter-provider-azure/pkg/controllers" "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" - "sigs.k8s.io/karpenter/pkg/controllers/state" coreoperator "sigs.k8s.io/karpenter/pkg/operator" corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" ) @@ -47,9 +46,10 @@ func main() { op. WithControllers(ctx, corecontrollers.NewControllers( + ctx, + op.Manager, op.Clock, op.GetClient(), - state.NewCluster(op.Clock, op.GetClient(), cloudProvider), op.EventRecorder, cloudProvider, )...). @@ -60,5 +60,6 @@ func main() { aksCloudProvider, op.InstanceProvider, )...). - Start(ctx) + //WithWebhooks(ctx, webhooks.NewWebhooks()...). + Start(ctx, cloudProvider) } diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 4f54b2e97..2005ac851 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -28,12 +28,9 @@ import ( controllers "github.com/Azure/karpenter-provider-azure/pkg/controllers" "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" - // Note the absence of corewebhooks: these pull in knative webhook-related packages and informers in init() // We don't give cluster-level roles when running in AKS managed mode, so their informers will produce errors and halt all other operations // corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" - - "sigs.k8s.io/karpenter/pkg/controllers/state" ) func main() { @@ -52,9 +49,10 @@ func main() { op. WithControllers(ctx, corecontrollers.NewControllers( + ctx, + op.Manager, op.Clock, op.GetClient(), - state.NewCluster(op.Clock, op.GetClient(), cloudProvider), op.EventRecorder, cloudProvider, )...). @@ -66,5 +64,5 @@ func main() { op.InstanceProvider, )...). // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). - Start(ctx) + Start(ctx, cloudProvider) } From 3b5d8d909e978ee0173702981d3bf2730e2ceb18 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:48:43 +0000 Subject: [PATCH 10/47] chore: update the alt operator --- .../karpenter-core/pkg/operator/operator.go | 93 ++++++++++++------- 1 file changed, 57 insertions(+), 36 deletions(-) diff --git a/pkg/alt/karpenter-core/pkg/operator/operator.go b/pkg/alt/karpenter-core/pkg/operator/operator.go index f920b075c..19c3266f7 100644 --- a/pkg/alt/karpenter-core/pkg/operator/operator.go +++ b/pkg/alt/karpenter-core/pkg/operator/operator.go @@ -30,34 +30,39 @@ import ( "github.com/go-logr/zapr" "github.com/samber/lo" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/events" coreoperator "sigs.k8s.io/karpenter/pkg/operator" - coreoperatorlogging "sigs.k8s.io/karpenter/pkg/operator/logging" coordinationv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/karpenter/pkg/operator/injection" "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" - //"k8s.io/apimachinery/pkg/fields" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/util/flowcontrol" "k8s.io/utils/clock" knativeinjection "knative.dev/pkg/injection" - knativelogging "knative.dev/pkg/logging" "knative.dev/pkg/signals" "knative.dev/pkg/system" - controllerruntime "sigs.k8s.io/controller-runtime" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/metrics/server" + + "sigs.k8s.io/karpenter/pkg/operator/logging" + + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/log" + + v1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) // Unmodified; for exposing private entity only @@ -101,12 +106,17 @@ func NewOperator() (context.Context, *coreoperator.Operator) { debug.SetMemoryLimit(newLimit) } + // Logging + logger := zapr.NewLogger(logging.NewLogger(ccPlaneCtx, component)) + log.SetLogger(logger) + klog.SetLogger(logger) + // Webhook // Unsupported -- skipping // Client Config ccPlaneConfig := lo.Must(rest.InClusterConfig()) - overlayConfig := controllerruntime.GetConfigOrDie() + overlayConfig := ctrl.GetConfigOrDie() ccPlaneConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(options.FromContext(ccPlaneCtx).KubeClientQPS), options.FromContext(ccPlaneCtx).KubeClientBurst) overlayConfig.RateLimiter = flowcontrol.NewTokenBucketRateLimiter(float32(options.FromContext(overlayCtx).KubeClientQPS), options.FromContext(overlayCtx).KubeClientBurst) // config.UserAgent = fmt.Sprintf("%s/%s", appName, Version) @@ -115,30 +125,22 @@ func NewOperator() (context.Context, *coreoperator.Operator) { // Client overlayKubernetesInterface := kubernetes.NewForConfigOrDie(overlayConfig) - //configMapWatcher := informer.NewInformedWatcher(ccPlaneKubernetesInterface, system.Namespace()) - //lo.Must0(configMapWatcher.Start(ccPlaneCtx.Done())) - - // Logging - logger := coreoperatorlogging.NewLogger(ccPlaneCtx, component) - ccPlaneCtx = knativelogging.WithLogger(ccPlaneCtx, logger) - overlayCtx = knativelogging.WithLogger(overlayCtx, logger) - coreoperatorlogging.ConfigureGlobalLoggers(ccPlaneCtx) // Manager - mgrOpts := controllerruntime.Options{ - Logger: ignoreDebugEvents(zapr.NewLogger(logger.Desugar())), - LeaderElection: options.FromContext(overlayCtx).EnableLeaderElection, - LeaderElectionID: "karpenter-leader-election", - LeaderElectionResourceLock: resourcelock.LeasesResourceLock, - Scheme: scheme.Scheme, + mgrOpts := ctrl.Options{ + Logger: logging.IgnoreDebugEvents(logger), + LeaderElection: !options.FromContext(overlayCtx).DisableLeaderElection, + LeaderElectionID: "karpenter-leader-election", + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, + LeaderElectionNamespace: system.Namespace(), + LeaderElectionReleaseOnCancel: true, + Scheme: scheme.Scheme, Metrics: server.Options{ BindAddress: fmt.Sprintf(":%d", options.FromContext(overlayCtx).MetricsPort), }, HealthProbeBindAddress: fmt.Sprintf(":%d", options.FromContext(overlayCtx).HealthProbePort), BaseContext: func() context.Context { - ctx := context.Background() - ctx = knativelogging.WithLogger(ctx, logger) - // ctx = injection.WithConfig(ctx, overlayConfig) + ctx := log.IntoContext(context.Background(), logger) ctx = injection.WithOptionsOrDie(ctx, options.Injectables...) return ctx }, @@ -164,31 +166,50 @@ func NewOperator() (context.Context, *coreoperator.Operator) { "/debug/pprof/threadcreate": pprof.Handler("threadcreate"), }) } - mgr, err := controllerruntime.NewManager(overlayConfig, mgrOpts) + mgr, err := ctrl.NewManager(overlayConfig, mgrOpts) mgr = lo.Must(mgr, err, "failed to setup manager") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.Pod{}, "spec.nodeName", func(o client.Object) []string { - return []string{o.(*v1.Pod).Spec.NodeName} + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &corev1.Pod{}, "spec.nodeName", func(o client.Object) []string { + return []string{o.(*corev1.Pod).Spec.NodeName} }), "failed to setup pod indexer") - lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.Node{}, "spec.providerID", func(o client.Object) []string { - return []string{o.(*v1.Node).Spec.ProviderID} + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &corev1.Node{}, "spec.providerID", func(o client.Object) []string { + return []string{o.(*corev1.Node).Spec.ProviderID} }), "failed to setup node provider id indexer") lo.Must0(func() error { _, _, err := lo.AttemptWithDelay(42, 10*time.Second, func(index int, duration time.Duration) error { - err := mgr.GetFieldIndexer().IndexField(overlayCtx, &v1beta1.NodeClaim{}, "status.providerID", func(o client.Object) []string { - return []string{o.(*v1beta1.NodeClaim).Status.ProviderID} + err := mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "status.providerID", func(o client.Object) []string { + return []string{o.(*v1.NodeClaim).Status.ProviderID} }) if err != nil { - knativelogging.FromContext(ccPlaneCtx).Infof("failed to setup machine provider id indexer, CRDs deployment may not be ready, index: %d, duration: %v, err: %v", index, duration, err) + log.FromContext(ccPlaneCtx).WithValues("index", index, "duration", duration, "err", err).Info("failed to setup NodeClaim provider id indexer, CRDs deployment may not be ready") } return err }) return err }(), "failed to setup nodeclaim provider id indexer, all attempts used") - lo.Must0(mgr.AddReadyzCheck("manager", func(req *http.Request) error { - return lo.Ternary(mgr.GetCache().WaitForCacheSync(req.Context()), nil, fmt.Errorf("failed to sync caches")) - })) + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.group", func(o client.Object) []string { + return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Group} + }), "failed to setup nodeclaim nodeclassref apiversion indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.kind", func(o client.Object) []string { + return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Kind} + }), "failed to setup nodeclaim nodeclassref kind indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { + return []string{o.(*v1.NodeClaim).Spec.NodeClassRef.Name} + }), "failed to setup nodeclaim nodeclassref name indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.group", func(o client.Object) []string { + return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Group} + }), "failed to setup nodepool nodeclassref apiversion indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.kind", func(o client.Object) []string { + return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Kind} + }), "failed to setup nodepool nodeclassref kind indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &v1.NodePool{}, "spec.template.spec.nodeClassRef.name", func(o client.Object) []string { + return []string{o.(*v1.NodePool).Spec.Template.Spec.NodeClassRef.Name} + }), "failed to setup nodepool nodeclassref name indexer") + lo.Must0(mgr.GetFieldIndexer().IndexField(overlayCtx, &storagev1.VolumeAttachment{}, "spec.nodeName", func(o client.Object) []string { + return []string{o.(*storagev1.VolumeAttachment).Spec.NodeName} + }), "failed to setup volumeattachment indexer") + lo.Must0(mgr.AddHealthzCheck("healthz", healthz.Ping)) lo.Must0(mgr.AddReadyzCheck("readyz", healthz.Ping)) From a66728a985439d61ce3ea7bc9b9be7a596d04fcd Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:52:30 +0000 Subject: [PATCH 11/47] chore: update the API (move kubelet config to AKSNodeClass) --- pkg/apis/apis.go | 22 +- .../karpenter.azure.com_aksnodeclasses.yaml | 291 +++++++-- pkg/apis/crds/karpenter.sh_nodeclaims.yaml | 430 ++++++++++++- pkg/apis/crds/karpenter.sh_nodepools.yaml | 565 +++++++++++++++++- pkg/apis/v1alpha2/aksnodeclass.go | 79 +++ pkg/apis/v1alpha2/aksnodeclass_status.go | 22 +- pkg/apis/v1alpha2/doc.go | 16 +- pkg/apis/v1alpha2/labels.go | 18 +- .../v1alpha2/nodepool_validation_cel_test.go | 49 +- pkg/apis/v1alpha2/suite_test.go | 5 +- pkg/apis/v1alpha2/zz_generated.deepcopy.go | 107 +++- 11 files changed, 1462 insertions(+), 142 deletions(-) diff --git a/pkg/apis/apis.go b/pkg/apis/apis.go index 472fde4de..cbf198ab7 100644 --- a/pkg/apis/apis.go +++ b/pkg/apis/apis.go @@ -20,28 +20,18 @@ package apis import ( _ "embed" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "k8s.io/apimachinery/pkg/runtime" + "github.com/awslabs/operatorpkg/object" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "github.com/samber/lo" - - "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "sigs.k8s.io/karpenter/pkg/apis" - "sigs.k8s.io/karpenter/pkg/utils/functional" -) - -var ( - // Builder includes all types within the apis package - Builder = runtime.NewSchemeBuilder( - v1alpha2.SchemeBuilder.AddToScheme, - ) - // AddToScheme may be used to add all resources defined in the project to a Scheme - AddToScheme = Builder.AddToScheme ) //go:generate controller-gen crd object:headerFile="../../hack/boilerplate.go.txt" paths="./..." output:crd:artifacts:config=crds var ( + Group = "karpenter.azure.com" + //CompatibilityGroup = "compatibility." + Group //go:embed crds/karpenter.azure.com_aksnodeclasses.yaml AKSNodeClassCRD []byte - CRDs = append(apis.CRDs, lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AKSNodeClassCRD))) + CRDs = append(apis.CRDs, + object.Unmarshal[apiextensionsv1.CustomResourceDefinition](AKSNodeClassCRD)) ) diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index 8cb872a0e..9b5afdeb9 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -3,76 +3,247 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.16.3 name: aksnodeclasses.karpenter.azure.com spec: group: karpenter.azure.com names: categories: - - karpenter + - karpenter kind: AKSNodeClass listKind: AKSNodeClassList plural: aksnodeclasses shortNames: - - aksnc - - aksncs + - aksnc + - aksncs singular: aksnodeclass scope: Cluster versions: - - name: v1alpha2 - schema: - openAPIV3Schema: - description: AKSNodeClass is the Schema for the AKSNodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. - This will contain configuration necessary to launch instances in AKS. - properties: - imageFamily: - default: Ubuntu2204 - description: ImageFamily is the image family that instances use. - enum: - - Ubuntu2204 - - AzureLinux - type: string - imageVersion: - description: ImageVersion is the image version that instances use. - type: string - osDiskSizeGB: - default: 128 - description: osDiskSizeGB is the size of the OS disk in GB. - format: int32 - minimum: 100 - type: integer - tags: - additionalProperties: + - name: v1alpha2 + schema: + openAPIV3Schema: + description: AKSNodeClass is the Schema for the AKSNodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. + This will contain configuration necessary to launch instances in AKS. + properties: + imageFamily: + default: Ubuntu2204 + description: ImageFamily is the image family that instances use. + enum: + - Ubuntu2204 + - AzureLinux type: string - description: Tags to be applied on Azure resources like instances. - type: object - type: object - status: - description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass - type: object - type: object - served: true - storage: true - subresources: - status: {} + imageVersion: + description: ImageVersion is the image version that instances use. + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + clusterDNS: + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. + items: + type: string + type: array + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + osDiskSizeGB: + default: 128 + description: osDiskSizeGB is the size of the OS disk in GB. + format: int32 + minimum: 100 + type: integer + tags: + additionalProperties: + type: string + description: Tags to be applied on Azure resources like instances. + type: object + type: object + status: + description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index 27a0f2cb1..780fd8420 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.16.3 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -20,6 +20,9 @@ spec: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type type: string + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + type: string - jsonPath: .metadata.labels.karpenter\.azure\.com/zone name: Zone type: string @@ -32,6 +35,367 @@ spec: - jsonPath: .metadata.creationTimestamp name: Age type: date + - jsonPath: .status.providerID + name: ID + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.azure.com" is restricted + rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + resources: + description: Resources models the resource requirements for the NodeClaim to launch + properties: + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object + type: object + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + x-kubernetes-validations: + - message: spec is immutable + rule: self == oldSelf + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node + type: object + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + lastPodEventTime: + description: |- + LastPodEventTime is updated with the last time a pod was scheduled + or removed from the node. A pod going terminal or terminating + is also considered as removed. + format: date-time + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date - jsonPath: .metadata.labels.karpenter\.sh/capacity-type name: Capacity priority: 1 @@ -219,8 +583,6 @@ spec: rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -250,13 +612,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ required: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -384,34 +747,52 @@ spec: conditions: description: Conditions contains signals for health and readiness items: - description: |- - Condition defines a readiness condition for a Knative resource. - See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties + description: Condition aliases the upstream type and adds additional helper methods properties: lastTransitionTime: description: |- - LastTransitionTime is the last time the condition transitioned from one status to another. - We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time type: string message: - description: A human readable message indicating details about the transition. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer reason: - description: The reason for the condition's last transition. - type: string - severity: description: |- - Severity with which to treat failures of this type of condition. - When this is not specified, it defaults to Error. + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + pattern: ^([A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?|)$ type: string status: - description: Status of the condition, one of True, False, Unknown. + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown type: string type: - description: Type of condition. + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string required: + - lastTransitionTime - status - type type: object @@ -430,6 +811,17 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 6595d6dd4..c5cb50744 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.14.0 + controller-gen.kubebuilder.io/version: v0.16.3 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -16,6 +16,493 @@ spec: singular: nodepool scope: Cluster versions: + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .status.resources.nodes + name: Nodes + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: integer + - jsonPath: .status.resources.cpu + name: CPU + priority: 1 + type: string + - jsonPath: .status.resources.memory + name: Memory + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. + properties: + disruption: + default: + consolidateAfter: 0s + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + budgets: + default: + - nodes: 10% + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. + items: + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. + properties: + duration: + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ + type: string + nodes: + default: 10% + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 + pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ + type: string + reasons: + description: |- + Reasons is a list of disruption methods that this budget applies to. If Reasons is not set, this budget applies to all methods. + Otherwise, this will apply to each reason defined. + allowed reasons are Underutilized, Empty, and Drifted. + items: + description: DisruptionReason defines valid reasons for disruption budgets. + enum: + - Underutilized + - Empty + - Drifted + type: string + type: array + schedule: + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. + pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ + type: string + required: + - nodes + type: object + maxItems: 50 + type: array + x-kubernetes-validations: + - message: '''schedule'' must be set with ''duration''' + rule: self.all(x, has(x.schedule) == has(x.duration)) + consolidateAfter: + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenEmptyOrUnderutilized + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenEmptyOrUnderutilized" if not specified + enum: + - WhenEmpty + - WhenEmptyOrUnderutilized + type: string + required: + - consolidateAfter + type: object + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations + type: object + labels: + additionalProperties: + type: string + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels + type: object + maxProperties: 100 + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self.all(x, x in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || x.find("^([^/]+)").endsWith("node.kubernetes.io") || x.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !x.find("^([^/]+)").endsWith("kubernetes.io")) + - message: label domain "k8s.io" is restricted + rule: self.all(x, x.find("^([^/]+)").endsWith("kops.k8s.io") || !x.find("^([^/]+)").endsWith("k8s.io")) + - message: label domain "karpenter.sh" is restricted + rule: self.all(x, x in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !x.find("^([^/]+)").endsWith("karpenter.sh")) + - message: label "karpenter.sh/nodepool" is restricted + rule: self.all(x, x != "karpenter.sh/nodepool") + - message: label "kubernetes.io/hostname" is restricted + rule: self.all(x, x != "kubernetes.io/hostname") + - message: label domain "karpenter.azure.com" is restricted + rule: self.all(x, x in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) + type: object + spec: + description: |- + NodeClaimTemplateSpec describes the desired state of the NodeClaim in the Nodepool + NodeClaimTemplateSpec is used in the NodePool's NodeClaimTemplate, with the resource requests omitted since + users are not able to set resource requests in the NodePool. + properties: + expireAfter: + default: 720h + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration + properties: + group: + description: API version of the referent + pattern: ^[^/]*$ + type: string + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: string + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: string + required: + - group + - kind + - name + type: object + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. + properties: + key: + description: The label key that the selector applies to. + type: string + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + x-kubernetes-validations: + - message: label domain "kubernetes.io" is restricted + rule: self in ["beta.kubernetes.io/instance-type", "failure-domain.beta.kubernetes.io/region", "beta.kubernetes.io/os", "beta.kubernetes.io/arch", "failure-domain.beta.kubernetes.io/zone", "topology.kubernetes.io/region", "node.kubernetes.io/instance-type", "kubernetes.io/arch", "kubernetes.io/os", "node.kubernetes.io/windows-build"] || self.find("^([^/]+)").endsWith("node.kubernetes.io") || self.find("^([^/]+)").endsWith("node-restriction.kubernetes.io") || !self.find("^([^/]+)").endsWith("kubernetes.io") + - message: label domain "k8s.io" is restricted + rule: self.find("^([^/]+)").endsWith("kops.k8s.io") || !self.find("^([^/]+)").endsWith("k8s.io") + - message: label domain "karpenter.sh" is restricted + rule: self in ["karpenter.sh/capacity-type", "karpenter.sh/nodepool"] || !self.find("^([^/]+)").endsWith("karpenter.sh") + - message: label "karpenter.sh/nodepool" is restricted + rule: self != "karpenter.sh/nodepool" + - message: label "kubernetes.io/hostname" is restricted + rule: self != "kubernetes.io/hostname" + - message: label domain "karpenter.azure.com" is restricted + rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + enum: + - In + - NotIn + - Exists + - DoesNotExist + - Gt + - Lt + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + required: + - key + - operator + type: object + maxItems: 100 + type: array + x-kubernetes-validations: + - message: requirements with operator 'In' must have a value defined + rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' + - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value + rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' + startupTaints: + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + enum: + - NoSchedule + - PreferNoSchedule + - NoExecute + key: + description: Required. The taint key to be applied to a node. + type: string + minLength: 1 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ + required: + - effect + - key + type: object + type: array + terminationGracePeriod: + description: |- + TerminationGracePeriod is the maximum duration the controller will wait before forcefully deleting the pods on a node, measured from when deletion is first initiated. + + Warning: this feature takes precedence over a Pod's terminationGracePeriodSeconds value, and bypasses any blocked PDBs or the karpenter.sh/do-not-disrupt annotation. + + This field is intended to be used by cluster administrators to enforce that nodes can be cycled within a given time period. + When set, drifted nodes will begin draining even if there are pods blocking eviction. Draining will respect PDBs and the do-not-disrupt annotation until the TGP is reached. + + Karpenter will preemptively delete pods so their terminationGracePeriodSeconds align with the node's terminationGracePeriod. + If a pod would be terminated without being granted its full terminationGracePeriodSeconds prior to the node timeout, + that pod will be deleted at T = node timeout - pod terminationGracePeriodSeconds. + + The feature can also be used to allow maximum time limits for long-running jobs which can delay node termination with preStop hooks. + If left undefined, the controller will wait indefinitely for pods to be drained. + pattern: ^([0-9]+(s|m|h))+$ + type: string + required: + - nodeClassRef + - requirements + type: object + required: + - spec + type: object + weight: + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + required: + - template + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass @@ -189,8 +676,6 @@ spec: rule: self.all(x, x != "karpenter.sh/nodepool") - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - - message: label domain "karpenter.azure.com" is restricted - rule: self.all(x, x in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !x.find("^([^/]+)").endsWith("karpenter.azure.com")) type: object spec: description: NodeClaimSpec describes the desired state of the NodeClaim @@ -347,8 +832,6 @@ spec: rule: self != "karpenter.sh/nodepool" - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - - message: label domain "karpenter.azure.com" is restricted - rule: self in [ "karpenter.azure.com/sku-name", "karpenter.azure.com/sku-family", "karpenter.azure.com/sku-version", "karpenter.azure.com/sku-cpu", "karpenter.azure.com/sku-memory", "karpenter.azure.com/sku-accelerator", "karpenter.azure.com/sku-networking-accelerated", "karpenter.azure.com/sku-storage-premium-capable", "karpenter.azure.com/sku-storage-ephemeralos-maxsize", "karpenter.azure.com/sku-encryptionathost-capable", "karpenter.azure.com/sku-gpu-name", "karpenter.azure.com/sku-gpu-manufacturer", "karpenter.azure.com/sku-gpu-count" ] || !self.find("^([^/]+)").endsWith("karpenter.azure.com") minValues: description: |- This field is ALPHA and can be dropped or replaced at any time @@ -378,13 +861,14 @@ spec: items: type: string type: array + x-kubernetes-list-type: atomic maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ required: - key - operator type: object - maxItems: 30 + maxItems: 100 type: array x-kubernetes-validations: - message: requirements with operator 'In' must have a value defined @@ -508,6 +992,62 @@ spec: status: description: NodePoolStatus defines the observed state of NodePool properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional helper methods + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array resources: additionalProperties: anyOf: @@ -522,6 +1062,17 @@ spec: - spec type: object served: true - storage: true + storage: false subresources: status: {} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: karpenter + namespace: kube-system + port: 8443 diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index ce9ea247a..916bbf264 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -44,11 +44,90 @@ type AKSNodeClassSpec struct { // Tags to be applied on Azure resources like instances. // +optional Tags map[string]string `json:"tags,omitempty"` + // Kubelet defines args to be used when configuring kubelet on provisioned nodes. + // They are a subset of the upstream types, recognizing not all options may be supported. + // Wherever possible, the types and names should reflect the upstream kubelet types. + // +kubebuilder:validation:XValidation:message="imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent",rule="has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true" + // +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true" + // +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true" + // +optional + Kubelet *KubeletConfiguration `json:"kubelet,omitempty" hash:"ignore"` +} + +// KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. +// They are a subset of the upstream types, recognizing not all options may be supported. +// Wherever possible, the types and names should reflect the upstream kubelet types. +// https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration +// https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 +type KubeletConfiguration struct { + // clusterDNS is a list of IP addresses for the cluster DNS server. + // Note that not all providers may use all addresses. + //+optional + ClusterDNS []string `json:"clusterDNS,omitempty"` + // MaxPods is an override for the maximum number of pods that can run on + // a worker node instance. + // +kubebuilder:validation:Minimum:=0 + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` + // PodsPerCore is an override for the number of pods that can run on a worker node + // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + // MaxPods is a lower value, that value will be used. + // +kubebuilder:validation:Minimum:=0 + // +optional + PodsPerCore *int32 `json:"podsPerCore,omitempty"` + // SystemReserved contains resources reserved for OS system daemons and kernel memory. + // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + SystemReserved map[string]string `json:"systemReserved,omitempty"` + // KubeReserved contains resources reserved for Kubernetes system components. + // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" + // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" + // +optional + KubeReserved map[string]string `json:"kubeReserved,omitempty"` + // EvictionHard is the map of signal names to quantities that define hard eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionHard map[string]string `json:"evictionHard,omitempty"` + // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoft map[string]string `json:"evictionSoft,omitempty"` + // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + // +kubebuilder:validation:XValidation:message="valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // +optional + EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` + // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + // response to soft eviction thresholds being met. + // +optional + EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // ImageGCHighThresholdPercent is the percent of disk usage after which image + // garbage collection is always run. The percent is calculated by dividing this + // field value by 100, so this field must be between 0 and 100, inclusive. + // When specified, the value must be greater than ImageGCLowThresholdPercent. + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCHighThresholdPercent *int32 `json:"imageGCHighThresholdPercent,omitempty"` + // ImageGCLowThresholdPercent is the percent of disk usage before which image + // garbage collection is never run. Lowest disk usage to garbage collect to. + // The percent is calculated by dividing this field value by 100, + // so the field value must be between 0 and 100, inclusive. + // When specified, the value must be less than imageGCHighThresholdPercent + // +kubebuilder:validation:Minimum:=0 + // +kubebuilder:validation:Maximum:=100 + // +optional + ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` + // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // +optional + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` } +// TODO: add hashing support // AKSNodeClass is the Schema for the AKSNodeClass API // +kubebuilder:object:root=true // +kubebuilder:resource:path=aksnodeclasses,scope=Cluster,categories=karpenter,shortName={aksnc,aksncs} +// +kubebuilder:storageversion // +kubebuilder:subresource:status type AKSNodeClass struct { metav1.TypeMeta `json:",inline"` diff --git a/pkg/apis/v1alpha2/aksnodeclass_status.go b/pkg/apis/v1alpha2/aksnodeclass_status.go index 9d93d5179..4ee13078c 100644 --- a/pkg/apis/v1alpha2/aksnodeclass_status.go +++ b/pkg/apis/v1alpha2/aksnodeclass_status.go @@ -16,7 +16,10 @@ limitations under the License. package v1alpha2 -import v1 "k8s.io/api/core/v1" +import ( + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" +) // Image contains resolved image selector values utilized for node launch type Image struct { @@ -25,9 +28,24 @@ type Image struct { ID string `json:"id"` // Requirements of the image to be utilized on an instance type // +required - Requirements []v1.NodeSelectorRequirement `json:"requirements"` + Requirements []corev1.NodeSelectorRequirement `json:"requirements"` } // AKSNodeClassStatus contains the resolved state of the AKSNodeClass type AKSNodeClassStatus struct { + // Conditions contains signals for health and readiness + // +optional + Conditions []status.Condition `json:"conditions,omitempty"` +} + +func (in *AKSNodeClass) StatusConditions() status.ConditionSet { + return status.NewReadyConditions().For(in) +} + +func (in *AKSNodeClass) GetConditions() []status.Condition { + return in.Status.Conditions +} + +func (in *AKSNodeClass) SetConditions(conditions []status.Condition) { + in.Status.Conditions = conditions } diff --git a/pkg/apis/v1alpha2/doc.go b/pkg/apis/v1alpha2/doc.go index 21c67ba76..ab759f70a 100644 --- a/pkg/apis/v1alpha2/doc.go +++ b/pkg/apis/v1alpha2/doc.go @@ -20,4 +20,18 @@ limitations under the License. // +groupName=karpenter.azure.com package v1alpha2 // doc.go is discovered by codegen -// TODO: tests +import ( + "github.com/Azure/karpenter-provider-azure/pkg/apis" + corev1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/scheme" +) + +func init() { + gv := schema.GroupVersion{Group: apis.Group, Version: "v1alpha2"} + corev1.AddToGroupVersion(scheme.Scheme, gv) + scheme.Scheme.AddKnownTypes(gv, + &AKSNodeClass{}, + &AKSNodeClassList{}, + ) +} diff --git a/pkg/apis/v1alpha2/labels.go b/pkg/apis/v1alpha2/labels.go index e1d5ff439..b80cbf023 100644 --- a/pkg/apis/v1alpha2/labels.go +++ b/pkg/apis/v1alpha2/labels.go @@ -18,13 +18,15 @@ package v1alpha2 import ( "k8s.io/apimachinery/pkg/util/sets" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/scheduling" + + "github.com/Azure/karpenter-provider-azure/pkg/apis" ) func init() { - corev1beta1.RestrictedLabelDomains = corev1beta1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) - corev1beta1.WellKnownLabels = corev1beta1.WellKnownLabels.Insert( + karpv1.RestrictedLabelDomains = karpv1.RestrictedLabelDomains.Insert(RestrictedLabelDomains...) + karpv1.WellKnownLabels = karpv1.WellKnownLabels.Insert( LabelSKUName, LabelSKUFamily, LabelSKUVersion, @@ -49,10 +51,11 @@ func init() { } var ( + TerminationFinalizer = apis.Group + "/termination" AzureToKubeArchitectures = map[string]string{ // TODO: consider using constants like compute.ArchitectureArm64 - "x64": corev1beta1.ArchitectureAmd64, - "Arm64": corev1beta1.ArchitectureArm64, + "x64": karpv1.ArchitectureAmd64, + "Arm64": karpv1.ArchitectureArm64, } RestrictedLabelDomains = []string{ Group, @@ -62,9 +65,8 @@ var ( LabelSKUHyperVGeneration, ) - AllowUndefinedLabels = func(options scheduling.CompatibilityOptions) scheduling.CompatibilityOptions { - options.AllowUndefined = corev1beta1.WellKnownLabels.Union(RestrictedLabels) - return options + AllowUndefinedWellKnownAndRestrictedLabels = func(options *scheduling.CompatibilityOptions) { + options.AllowUndefined = karpv1.WellKnownLabels.Union(RestrictedLabels) } // alternative zone label for Machine (the standard one is protected for AKS nodes) diff --git a/pkg/apis/v1alpha2/nodepool_validation_cel_test.go b/pkg/apis/v1alpha2/nodepool_validation_cel_test.go index 822aba220..a74fd714e 100644 --- a/pkg/apis/v1alpha2/nodepool_validation_cel_test.go +++ b/pkg/apis/v1alpha2/nodepool_validation_cel_test.go @@ -23,34 +23,35 @@ import ( "github.com/Pallinder/go-randomdata" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) var _ = Describe("CEL/Validation", func() { - var nodePool *v1beta1.NodePool + var nodePool *karpv1.NodePool BeforeEach(func() { if env.Version.Minor() < 25 { Skip("CEL Validation is for 1.25>") } - nodePool = &v1beta1.NodePool{ + nodePool = &karpv1.NodePool{ ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - Spec: v1beta1.NodePoolSpec{ - Template: v1beta1.NodeClaimTemplate{ - Spec: v1beta1.NodeClaimSpec{ - NodeClassRef: &v1beta1.NodeClassReference{ - Kind: "NodeClaim", - Name: "default", + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: "karpenter.azure.com", + Kind: "AKSNodeClass", + Name: "default", }, - Requirements: []v1beta1.NodeSelectorRequirementWithMinValues{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpExists, }, }, }, @@ -62,9 +63,9 @@ var _ = Describe("CEL/Validation", func() { Context("Requirements", func() { It("should allow restricted domains exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.LabelDomainExceptions { - nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + for label := range karpv1.LabelDomainExceptions { + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label + "/test", Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) @@ -74,9 +75,9 @@ var _ = Describe("CEL/Validation", func() { }) It("should allow well known label exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { - nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey)) { + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label, Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) @@ -87,8 +88,8 @@ var _ = Describe("CEL/Validation", func() { It("should not allow internal labels", func() { oldNodePool := nodePool.DeepCopy() for label := range v1alpha2.RestrictedLabels { - nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: corev1.NodeSelectorRequirement{Key: label, Operator: corev1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).ToNot(Succeed()) @@ -100,7 +101,7 @@ var _ = Describe("CEL/Validation", func() { Context("Labels", func() { It("should allow restricted domains exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.LabelDomainExceptions { + for label := range karpv1.LabelDomainExceptions { nodePool.Spec.Template.Labels = map[string]string{ label: "test", } @@ -112,7 +113,7 @@ var _ = Describe("CEL/Validation", func() { }) It("should allow well known label exceptions", func() { oldNodePool := nodePool.DeepCopy() - for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { + for label := range karpv1.WellKnownLabels.Difference(sets.New(karpv1.NodePoolLabelKey)) { nodePool.Spec.Template.Labels = map[string]string{ label: "test", } diff --git a/pkg/apis/v1alpha2/suite_test.go b/pkg/apis/v1alpha2/suite_test.go index 331360ccb..ff3674aae 100644 --- a/pkg/apis/v1alpha2/suite_test.go +++ b/pkg/apis/v1alpha2/suite_test.go @@ -20,13 +20,14 @@ import ( "context" "testing" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/Azure/karpenter-provider-azure/pkg/apis" @@ -44,7 +45,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) azureEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/apis/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/v1alpha2/zz_generated.deepcopy.go index 4b5ecdea5..920a3579e 100644 --- a/pkg/apis/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha2/zz_generated.deepcopy.go @@ -1,3 +1,5 @@ +//go:build !ignore_autogenerated + /* Portions Copyright (c) Microsoft Corporation. @@ -19,7 +21,9 @@ limitations under the License. package v1alpha2 import ( - "k8s.io/api/core/v1" + "github.com/awslabs/operatorpkg/status" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -29,7 +33,7 @@ func (in *AKSNodeClass) DeepCopyInto(out *AKSNodeClass) { out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status + in.Status.DeepCopyInto(&out.Status) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClass. @@ -112,6 +116,11 @@ func (in *AKSNodeClassSpec) DeepCopyInto(out *AKSNodeClassSpec) { (*out)[key] = val } } + if in.Kubelet != nil { + in, out := &in.Kubelet, &out.Kubelet + *out = new(KubeletConfiguration) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassSpec. @@ -127,6 +136,13 @@ func (in *AKSNodeClassSpec) DeepCopy() *AKSNodeClassSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AKSNodeClassStatus) DeepCopyInto(out *AKSNodeClassStatus) { *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]status.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassStatus. @@ -144,7 +160,7 @@ func (in *Image) DeepCopyInto(out *Image) { *out = *in if in.Requirements != nil { in, out := &in.Requirements, &out.Requirements - *out = make([]v1.NodeSelectorRequirement, len(*in)) + *out = make([]corev1.NodeSelectorRequirement, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -160,3 +176,88 @@ func (in *Image) DeepCopy() *Image { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { + *out = *in + if in.ClusterDNS != nil { + in, out := &in.ClusterDNS, &out.ClusterDNS + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } + if in.PodsPerCore != nil { + in, out := &in.PodsPerCore, &out.PodsPerCore + *out = new(int32) + **out = **in + } + if in.SystemReserved != nil { + in, out := &in.SystemReserved, &out.SystemReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.KubeReserved != nil { + in, out := &in.KubeReserved, &out.KubeReserved + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionHard != nil { + in, out := &in.EvictionHard, &out.EvictionHard + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoft != nil { + in, out := &in.EvictionSoft, &out.EvictionSoft + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionSoftGracePeriod != nil { + in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod + *out = make(map[string]v1.Duration, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.EvictionMaxPodGracePeriod != nil { + in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod + *out = new(int32) + **out = **in + } + if in.ImageGCHighThresholdPercent != nil { + in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent + *out = new(int32) + **out = **in + } + if in.ImageGCLowThresholdPercent != nil { + in, out := &in.ImageGCLowThresholdPercent, &out.ImageGCLowThresholdPercent + *out = new(int32) + **out = **in + } + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. +func (in *KubeletConfiguration) DeepCopy() *KubeletConfiguration { + if in == nil { + return nil + } + out := new(KubeletConfiguration) + in.DeepCopyInto(out) + return out +} From 9431774766ec1fe55eb001eef22b0cf58c3468f0 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:54:36 +0000 Subject: [PATCH 12/47] chore: migrate cloud provider to v1 API --- pkg/cache/unavailableofferings.go | 8 +- pkg/cloudprovider/cloudprovider.go | 121 +++++++++++++++++++---------- pkg/cloudprovider/drift.go | 6 +- pkg/cloudprovider/events/events.go | 16 ++-- pkg/cloudprovider/suite_test.go | 90 +++++++++++---------- 5 files changed, 142 insertions(+), 99 deletions(-) diff --git a/pkg/cache/unavailableofferings.go b/pkg/cache/unavailableofferings.go index e6cc499c5..a42c97244 100644 --- a/pkg/cache/unavailableofferings.go +++ b/pkg/cache/unavailableofferings.go @@ -24,11 +24,11 @@ import ( "github.com/patrickmn/go-cache" "knative.dev/pkg/logging" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) var ( - spotKey = key("", "", v1beta1.CapacityTypeSpot) + spotKey = key("", "", karpv1.CapacityTypeSpot) ) // UnavailableOfferings stores any offerings that return ICE (insufficient capacity errors) when @@ -58,7 +58,7 @@ func NewUnavailableOfferings() *UnavailableOfferings { // IsUnavailable returns true if the offering appears in the cache func (u *UnavailableOfferings) IsUnavailable(instanceType, zone, capacityType string) bool { - if capacityType == v1beta1.CapacityTypeSpot { + if capacityType == karpv1.CapacityTypeSpot { if _, found := u.cache.Get(spotKey); found { return true } @@ -69,7 +69,7 @@ func (u *UnavailableOfferings) IsUnavailable(instanceType, zone, capacityType st // MarkSpotUnavailable communicates recently observed temporary capacity shortages for spot func (u *UnavailableOfferings) MarkSpotUnavailableWithTTL(ctx context.Context, ttl time.Duration) { - u.MarkUnavailableWithTTL(ctx, "SpotUnavailable", "", "", v1beta1.CapacityTypeSpot, UnavailableOfferingsTTL) + u.MarkUnavailableWithTTL(ctx, "SpotUnavailable", "", "", karpv1.CapacityTypeSpot, UnavailableOfferingsTTL) } // MarkUnavailableWithTTL allows us to mark an offering unavailable with a custom TTL diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 047376fbd..b628220cb 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -18,9 +18,13 @@ package cloudprovider import ( "context" + stderrors "errors" "fmt" "net/http" "strings" + "time" + + "github.com/awslabs/operatorpkg/status" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -35,6 +39,7 @@ import ( // nolint SA1019 - deprecated package "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/inplaceupdate" @@ -45,27 +50,27 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/samber/lo" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + coreapis "sigs.k8s.io/karpenter/pkg/apis" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" - "sigs.k8s.io/karpenter/pkg/utils/functional" "sigs.k8s.io/karpenter/pkg/utils/resources" ) var _ cloudprovider.CloudProvider = (*CloudProvider)(nil) type CloudProvider struct { - instanceTypeProvider *instancetype.Provider - instanceProvider *instance.Provider + instanceTypeProvider instancetype.Provider + instanceProvider instance.Provider kubeClient client.Client imageProvider *imagefamily.Provider recorder events.Recorder } -func New(instanceTypeProvider *instancetype.Provider, instanceProvider *instance.Provider, recorder events.Recorder, +func New(instanceTypeProvider instancetype.Provider, instanceProvider instance.Provider, recorder events.Recorder, kubeClient client.Client, imageProvider *imagefamily.Provider) *CloudProvider { return &CloudProvider{ instanceTypeProvider: instanceTypeProvider, @@ -77,7 +82,7 @@ func New(instanceTypeProvider *instancetype.Provider, instanceProvider *instance } // Create a node given the constraints. -func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*corev1beta1.NodeClaim, error) { +func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim) (*karpv1.NodeClaim, error) { nodeClass, err := c.resolveNodeClassFromNodeClaim(ctx, nodeClaim) if err != nil { if errors.IsNotFound(err) { @@ -87,6 +92,25 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC return nil, cloudprovider.NewInsufficientCapacityError(fmt.Errorf("resolving node class, %w", err)) } + /* + // TODO: Remove this after v1 + nodePool, err := utils.ResolveNodePoolFromNodeClaim(ctx, c.kubeClient, nodeClaim) + if err != nil { + return nil, err + } + kubeletHash, err := utils.GetHashKubelet(nodePool, nodeClass) + if err != nil { + return nil, err + } + */ + nodeClassReady := nodeClass.StatusConditions().Get(status.ConditionReady) + if nodeClassReady.IsFalse() { + return nil, cloudprovider.NewNodeClassNotReadyError(stderrors.New(nodeClassReady.Message)) + } + if nodeClassReady.IsUnknown() { + return nil, fmt.Errorf("resolving NodeClass readiness, NodeClass is in Ready=Unknown, %s", nodeClassReady.Message) + } + instanceTypes, err := c.resolveInstanceTypes(ctx, nodeClaim, nodeClass) if err != nil { return nil, fmt.Errorf("resolving instance types, %w", err) @@ -102,15 +126,18 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC return i.Name == string(lo.FromPtr(instance.Properties.HardwareProfile.VMSize)) }) - return c.instanceToNodeClaim(ctx, instance, instanceType) + nc, err := c.instanceToNodeClaim(ctx, instance, instanceType) + + // TODO: record nodeclass hash & version + return nc, err } -func (c *CloudProvider) List(ctx context.Context) ([]*corev1beta1.NodeClaim, error) { +func (c *CloudProvider) List(ctx context.Context) ([]*karpv1.NodeClaim, error) { instances, err := c.instanceProvider.List(ctx) if err != nil { return nil, fmt.Errorf("listing instances, %w", err) } - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim for _, instance := range instances { instanceType, err := c.resolveInstanceTypeFromInstance(ctx, instance) if err != nil { @@ -126,7 +153,7 @@ func (c *CloudProvider) List(ctx context.Context) ([]*corev1beta1.NodeClaim, err return nodeClaims, nil } -func (c *CloudProvider) Get(ctx context.Context, providerID string) (*corev1beta1.NodeClaim, error) { +func (c *CloudProvider) Get(ctx context.Context, providerID string) (*karpv1.NodeClaim, error) { vmName, err := utils.GetVMName(providerID) if err != nil { return nil, fmt.Errorf("getting vm name, %w", err) @@ -148,11 +175,7 @@ func (c *CloudProvider) LivenessProbe(req *http.Request) error { } // GetInstanceTypes returns all available InstanceTypes -func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *corev1beta1.NodePool) ([]*cloudprovider.InstanceType, error) { - if nodePool == nil { - return c.instanceTypeProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, &v1alpha2.AKSNodeClass{}) - } - +func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *karpv1.NodePool) ([]*cloudprovider.InstanceType, error) { nodeClass, err := c.resolveNodeClassFromNodePool(ctx, nodePool) if err != nil { if errors.IsNotFound(err) { @@ -163,14 +186,14 @@ func (c *CloudProvider) GetInstanceTypes(ctx context.Context, nodePool *corev1be // as the cause. return nil, fmt.Errorf("resolving node class, %w", err) } - instanceTypes, err := c.instanceTypeProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + instanceTypes, err := c.instanceTypeProvider.List(ctx, nodeClass) if err != nil { return nil, err } return instanceTypes, nil } -func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) error { +func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *karpv1.NodeClaim) error { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("nodeclaim", nodeClaim.Name)) vmName, err := utils.GetVMName(nodeClaim.Status.ProviderID) @@ -180,7 +203,7 @@ func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *corev1beta1.NodeC return c.instanceProvider.Delete(ctx, vmName) } -func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) { +func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim) (cloudprovider.DriftReason, error) { if nodeClaim.Spec.NodeClassRef == nil { return "", nil } @@ -214,48 +237,46 @@ func (c *CloudProvider) Name() string { return "azure" } -func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { - return []schema.GroupVersionKind{ - { - Group: v1alpha2.SchemeGroupVersion.Group, - Version: v1alpha2.SchemeGroupVersion.Version, - Kind: "AKSNodeClass", - }, - } +func (c *CloudProvider) GetSupportedNodeClasses() []status.Object { + return []status.Object{&v1alpha2.AKSNodeClass{}} } -func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*v1alpha2.AKSNodeClass, error) { +func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *karpv1.NodeClaim) (*v1alpha2.AKSNodeClass, error) { nodeClass := &v1alpha2.AKSNodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeClaim.Spec.NodeClassRef.Name}, nodeClass); err != nil { return nil, err } // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound if !nodeClass.DeletionTimestamp.IsZero() { - return nil, errors.NewNotFound(v1alpha2.SchemeGroupVersion.WithResource("aksnodeclasses").GroupResource(), nodeClass.Name) + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound, + // but we return a different error message to be clearer to users + return nil, newTerminatingNodeClassError(nodeClass.Name) } return nodeClass, nil } -func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePool *corev1beta1.NodePool) (*v1alpha2.AKSNodeClass, error) { +func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePool *karpv1.NodePool) (*v1alpha2.AKSNodeClass, error) { nodeClass := &v1alpha2.AKSNodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePool.Spec.Template.Spec.NodeClassRef.Name}, nodeClass); err != nil { return nil, err } // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound if !nodeClass.DeletionTimestamp.IsZero() { - return nil, errors.NewNotFound(v1alpha2.SchemeGroupVersion.WithResource("aksnodeclasses").GroupResource(), nodeClass.Name) + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound, + // but we return a different error message to be clearer to users + return nil, newTerminatingNodeClassError(nodeClass.Name) } return nodeClass, nil } -func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { - instanceTypes, err := c.instanceTypeProvider.List(ctx, nodeClaim.Spec.Kubelet, nodeClass) +func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { + instanceTypes, err := c.instanceTypeProvider.List(ctx, nodeClass) if err != nil { return nil, fmt.Errorf("getting instance types, %w", err) } reqs := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) return lo.Filter(instanceTypes, func(i *cloudprovider.InstanceType, _ int) bool { - return reqs.Compatible(i.Requirements, v1alpha2.AllowUndefinedLabels) == nil && + return reqs.Compatible(i.Requirements, v1alpha2.AllowUndefinedWellKnownAndRestrictedLabels) == nil && len(i.Offerings.Compatible(reqs).Available()) > 0 && resources.Fits(nodeClaim.Spec.Resources.Requests, i.Allocatable()) }), nil @@ -284,30 +305,31 @@ func (c *CloudProvider) resolveInstanceTypeFromInstance(ctx context.Context, ins return instanceType, nil } -func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instance *armcompute.VirtualMachine) (*corev1beta1.NodePool, error) { - nodePoolName, ok := instance.Tags[corev1beta1.NodePoolLabelKey] +func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instance *armcompute.VirtualMachine) (*karpv1.NodePool, error) { + nodePoolName, ok := instance.Tags[karpv1.NodePoolLabelKey] if ok && *nodePoolName != "" { - nodePool := &corev1beta1.NodePool{} + nodePool := &karpv1.NodePool{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: *nodePoolName}, nodePool); err != nil { return nil, err } return nodePool, nil } - return nil, errors.NewNotFound(schema.GroupResource{Group: corev1beta1.Group, Resource: "NodePool"}, "") + return nil, errors.NewNotFound(schema.GroupResource{Group: coreapis.Group, Resource: "nodepools"}, "") } -func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute.VirtualMachine, instanceType *cloudprovider.InstanceType) (*corev1beta1.NodeClaim, error) { - nodeClaim := &corev1beta1.NodeClaim{} +func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute.VirtualMachine, instanceType *cloudprovider.InstanceType) (*karpv1.NodeClaim, error) { + nodeClaim := &karpv1.NodeClaim{} labels := map[string]string{} annotations := map[string]string{} if instanceType != nil { labels = instance.GetAllSingleValuedRequirementLabels(instanceType) - nodeClaim.Status.Capacity = functional.FilterMap(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) - nodeClaim.Status.Allocatable = functional.FilterMap(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + nodeClaim.Status.Capacity = lo.PickBy(instanceType.Capacity, func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) + nodeClaim.Status.Allocatable = lo.PickBy(instanceType.Allocatable(), func(_ v1.ResourceName, v resource.Quantity) bool { return !resources.IsZero(v) }) } + // TODO: review logic for determining zone (AWS uses Zone from subnet resolved and aviailable from NodeClass conditions ...) if zoneID, err := instance.GetZoneID(vm); err != nil { logging.FromContext(ctx).Warnf("Failed to get zone for VM %s, %v", *vm.Name, err) } else { @@ -316,11 +338,11 @@ func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute. labels[v1alpha2.AlternativeLabelTopologyZone] = zone } - labels[corev1beta1.CapacityTypeLabelKey] = instance.GetCapacityType(vm) + labels[karpv1.CapacityTypeLabelKey] = instance.GetCapacityType(vm) // TODO: v1beta1 new kes/labels if tag, ok := vm.Tags[instance.NodePoolTagKey]; ok { - labels[corev1beta1.NodePoolLabelKey] = *tag + labels[karpv1.NodePoolLabelKey] = *tag } inPlaceUpdateHash, err := inplaceupdate.HashFromVM(vm) @@ -333,7 +355,12 @@ func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute. nodeClaim.Labels = labels nodeClaim.Annotations = annotations nodeClaim.CreationTimestamp = metav1.Time{Time: *vm.Properties.TimeCreated} + // Set the deletionTimestamp to be the current time if the instance is currently terminating + if utils.IsVMDeleting(*vm) { + nodeClaim.DeletionTimestamp = &metav1.Time{Time: time.Now()} + } nodeClaim.Status.ProviderID = utils.ResourceIDToProviderID(ctx, *vm.ID) + //TOFIX (could be nil, at least when faked) nodeClaim.Status.ImageID = utils.ImageReferenceToString(*vm.Properties.StorageProfile.ImageReference) return nodeClaim, nil } @@ -348,3 +375,11 @@ func makeZone(location string, zoneID string) string { } return fmt.Sprintf("%s-%s", strings.ToLower(location), zoneID) } + +// newTerminatingNodeClassError returns a NotFound error for handling by +func newTerminatingNodeClassError(name string) *errors.StatusError { + qualifiedResource := schema.GroupResource{Group: apis.Group, Resource: "aksnodeclasses"} + err := errors.NewNotFound(qualifiedResource, name) + err.ErrStatus.Message = fmt.Sprintf("%s %q is terminating, treating as not found", qualifiedResource.String(), name) + return err +} diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index de150f494..1db475d08 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -31,7 +31,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" ) @@ -40,7 +40,7 @@ const ( ImageVersionDrift cloudprovider.DriftReason = "ImageVersionDrift" ) -func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (cloudprovider.DriftReason, error) { +func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim) (cloudprovider.DriftReason, error) { logger := logging.FromContext(ctx) nodeName := nodeClaim.Status.NodeName @@ -73,7 +73,7 @@ func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *core // Feel reassessing this within the future with a potential minor refactor would be best to fix the gocyclo. // nolint: gocyclo func (c *CloudProvider) isImageVersionDrifted( - ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { + ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { logger := logging.FromContext(ctx) id, err := utils.GetVMName(nodeClaim.Status.ProviderID) diff --git a/pkg/cloudprovider/events/events.go b/pkg/cloudprovider/events/events.go index 5a961866b..5b745eae6 100644 --- a/pkg/cloudprovider/events/events.go +++ b/pkg/cloudprovider/events/events.go @@ -17,26 +17,26 @@ limitations under the License. package events import ( - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + v1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/events" ) -func NodePoolFailedToResolveNodeClass(nodePool *v1beta1.NodePool) events.Event { +func NodePoolFailedToResolveNodeClass(nodePool *v1.NodePool) events.Event { return events.Event{ InvolvedObject: nodePool, - Type: v1.EventTypeWarning, - Message: "Failed resolving AKSNodeClass", + Type: corev1.EventTypeWarning, + Message: "Failed resolving NodeClass", DedupeValues: []string{string(nodePool.UID)}, } } -func NodeClaimFailedToResolveNodeClass(nodeClaim *v1beta1.NodeClaim) events.Event { +func NodeClaimFailedToResolveNodeClass(nodeClaim *v1.NodeClaim) events.Event { return events.Event{ InvolvedObject: nodeClaim, - Type: v1.EventTypeWarning, - Message: "Failed resolving AKSNodeClass", + Type: corev1.EventTypeWarning, + Message: "Failed resolving NodeClass", DedupeValues: []string{string(nodeClaim.UID)}, } } diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index d7d13be51..e374fb004 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -20,10 +20,13 @@ package cloudprovider import ( "context" "testing" + "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + "github.com/awslabs/operatorpkg/object" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,14 +34,14 @@ import ( clock "k8s.io/utils/clock/testing" "github.com/Azure/karpenter-provider-azure/pkg/utils" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + opstatus "github.com/awslabs/operatorpkg/status" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "knative.dev/pkg/logging/testing" @@ -55,32 +58,33 @@ var ctx context.Context var stop context.CancelFunc var env *coretest.Environment var azureEnv *test.Environment +var prov *provisioning.Provisioner +var cloudProvider *CloudProvider +var cluster *state.Cluster var fakeClock *clock.FakeClock -var coreProvisioner *provisioning.Provisioner +var recorder events.Recorder -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool var nodeClass *v1alpha2.AKSNodeClass -var nodeClaim *corev1beta1.NodeClaim -var cluster *state.Cluster -var cloudProvider *CloudProvider +var nodeClaim *karpv1.NodeClaim func TestCloudProvider(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "CloudProvider") + RunSpecs(t, "cloudProvider/Azure") } var _ = BeforeSuite(func() { - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coreoptions.ToContext(ctx, coretest.Options(coretest.OptionsFields{FeatureGates: coretest.FeatureGates{Drift: lo.ToPtr(true)}})) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) + ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) azureEnv = test.NewEnvironment(ctx, env) - - fakeClock = &clock.FakeClock{} - cloudProvider = New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + fakeClock = clock.NewFakeClock(time.Now()) + recorder = events.NewRecorder(&record.FakeRecorder{}) + cloudProvider = New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, recorder, env.Client, azureEnv.ImageProvider) + cluster = state.NewCluster(fakeClock, env.Client) + prov = provisioning.NewProvisioner(env.Client, recorder, cloudProvider, cluster) }) var _ = AfterSuite(func() { @@ -90,28 +94,32 @@ var _ = AfterSuite(func() { var _ = BeforeEach(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) - // TODO v1beta1 options - // ctx = options.ToContext(ctx, test.Options()) ctx = options.ToContext(ctx, test.Options()) + nodeClass = test.AKSNodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClass.StatusConditions().SetTrue(opstatus.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, }, }) - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{corev1beta1.NodePoolLabelKey: nodePool.Name}, + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }) @@ -127,8 +135,8 @@ var _ = AfterEach(func() { var _ = Describe("CloudProvider", func() { It("should list nodeclaim created by the CloudProvider", func() { ExpectApplied(ctx, env.Client, nodeClass, nodePool) - pod := coretest.UnschedulablePod(coretest.PodOptions{}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) @@ -143,7 +151,7 @@ var _ = Describe("CloudProvider", func() { }) It("should return an ICE error when there are no instance types to launch", func() { // Specify no instance types and expect to receive a capacity error - nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ { NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, @@ -159,7 +167,7 @@ var _ = Describe("CloudProvider", func() { Expect(cloudProviderMachine).To(BeNil()) }) Context("Drift", func() { - var nodeClaim *corev1beta1.NodeClaim + var nodeClaim *karpv1.NodeClaim var pod *v1.Pod BeforeEach(func() { instanceType := "Standard_D2_v2" @@ -167,7 +175,7 @@ var _ = Describe("CloudProvider", func() { pod = coretest.UnschedulablePod(coretest.PodOptions{ NodeSelector: map[string]string{v1.LabelInstanceTypeStable: instanceType}, }) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) @@ -176,18 +184,18 @@ var _ = Describe("CloudProvider", func() { vmName := input.VMName // Corresponding NodeClaim - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: utils.ResourceIDToProviderID(ctx, utils.MkVMID(rg, vmName)), }, ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: instanceType, + karpv1.NodePoolLabelKey: nodePool.Name, + v1.LabelInstanceTypeStable: instanceType, }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -211,7 +219,7 @@ var _ = Describe("CloudProvider", func() { Expect(drifted).To(BeEmpty()) }) It("should error drift if NodeClaim doesn't have provider id", func() { - nodeClaim.Status = corev1beta1.NodeClaimStatus{} + nodeClaim.Status = karpv1.NodeClaimStatus{} drifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).To(HaveOccurred()) Expect(drifted).To(BeEmpty()) From 53a24a7b8358bd76e056a2baa88ffb5432329edd Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:55:58 +0000 Subject: [PATCH 13/47] chore: migrate operator to v1 API --- pkg/operator/operator.go | 17 ++++++++--------- pkg/operator/options/options.go | 3 ++- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 4f7e70276..cee12dbd0 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -27,12 +27,11 @@ import ( "k8s.io/client-go/rest" "k8s.io/client-go/transport" "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/scheme" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" - "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/auth" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" @@ -48,8 +47,8 @@ import ( ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) + karpv1beta1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) } type Operator struct { @@ -61,8 +60,8 @@ type Operator struct { ImageResolver *imagefamily.Resolver LaunchTemplateProvider *launchtemplate.Provider PricingProvider *pricing.Provider - InstanceTypesProvider *instancetype.Provider - InstanceProvider *instance.Provider + InstanceTypesProvider instancetype.Provider + InstanceProvider *instance.DefaultProvider LoadBalancerProvider *loadbalancer.Provider } @@ -107,7 +106,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont azConfig.Location, vnetGUID, ) - instanceTypeProvider := instancetype.NewProvider( + instanceTypeProvider := instancetype.NewDefaultProvider( azConfig.Location, cache.New(instancetype.InstanceTypesCacheTTL, azurecache.DefaultCleanupInterval), azClient.SKUClient, @@ -119,7 +118,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont cache.New(loadbalancer.LoadBalancersCacheTTL, azurecache.DefaultCleanupInterval), azConfig.NodeResourceGroup, ) - instanceProvider := instance.NewProvider( + instanceProvider := instance.NewDefaultProvider( azClient, instanceTypeProvider, launchTemplateProvider, diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index d3a3acac8..273170e00 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -27,6 +27,7 @@ import ( "os" "strings" + "github.com/Azure/karpenter-provider-azure/pkg/utils" "k8s.io/apimachinery/pkg/util/sets" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" "sigs.k8s.io/karpenter/pkg/utils/env" @@ -76,7 +77,7 @@ type Options struct { func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource tags.") fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "[REQUIRED] The external kubernetes cluster endpoint for new nodes to connect with.") - fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", utils.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.KubeletClientTLSBootstrapToken, "kubelet-bootstrap-token", env.WithDefaultString("KUBELET_BOOTSTRAP_TOKEN", ""), "[REQUIRED] The bootstrap token for new nodes to join the cluster.") fs.StringVar(&o.SSHPublicKey, "ssh-public-key", env.WithDefaultString("SSH_PUBLIC_KEY", ""), "[REQUIRED] VM SSH public key.") fs.StringVar(&o.NetworkPlugin, "network-plugin", env.WithDefaultString("NETWORK_PLUGIN", "azure"), "The network plugin used by the cluster.") From 0f7350427c86c4552b749ad4577b36be73b0e125 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:56:54 +0000 Subject: [PATCH 14/47] chore: migrate controllers to v1 API --- pkg/controllers/controllers.go | 13 +-- .../nodeclaim/garbagecollection/controller.go | 48 ++++++----- .../nodeclaim/garbagecollection/suite_test.go | 81 ++++++++++--------- .../nodeclaim/inplaceupdate/controller.go | 59 +++++++------- .../nodeclaim/inplaceupdate/suite_test.go | 61 +++++++------- .../nodeclaim/inplaceupdate/utils.go | 4 +- 6 files changed, 138 insertions(+), 128 deletions(-) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 8da248639..2758c9f51 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -19,22 +19,23 @@ package controllers import ( "context" - "knative.dev/pkg/logging" + "github.com/awslabs/operatorpkg/controller" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/karpenter/pkg/operator/controller" - "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" nodeclaimgarbagecollection "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/garbagecollection" + nodeclassstatus "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/status" + "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/inplaceupdate" "github.com/Azure/karpenter-provider-azure/pkg/providers/instance" - "github.com/Azure/karpenter-provider-azure/pkg/utils/project" ) -func NewControllers(ctx context.Context, kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider, instanceProvider *instance.Provider) []controller.Controller { - logging.FromContext(ctx).With("version", project.Version).Debugf("discovered version") +func NewControllers(ctx context.Context, kubeClient client.Client, cloudProvider cloudprovider.CloudProvider, instanceProvider instance.Provider) []controller.Controller { controllers := []controller.Controller{ nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), inplaceupdate.NewController(kubeClient, instanceProvider), + nodeclassstatus.NewController(kubeClient), } return controllers } diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index ba2e4568c..e3a5bc93c 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -21,30 +21,33 @@ import ( "fmt" "time" - "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" + "github.com/awslabs/operatorpkg/singleton" + + // "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" "github.com/samber/lo" "go.uber.org/multierr" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/util/workqueue" "knative.dev/pkg/logging" + controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" - "sigs.k8s.io/karpenter/pkg/operator/controller" ) type Controller struct { kubeClient client.Client - cloudProvider *cloudprovider.CloudProvider + cloudProvider corecloudprovider.CloudProvider successfulCount uint64 // keeps track of successful reconciles for more aggressive requeueing near the start of the controller } -func NewController(kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider) *Controller { +func NewController(kubeClient client.Client, cloudProvider corecloudprovider.CloudProvider) *Controller { return &Controller{ kubeClient: kubeClient, cloudProvider: cloudProvider, @@ -52,11 +55,9 @@ func NewController(kubeClient client.Client, cloudProvider *cloudprovider.CloudP } } -func (c *Controller) Name() string { - return "nodeclaim.garbagecollection" -} +func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclaim.garbagecollection") -func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { // We LIST VMs on the CloudProvider BEFORE we grab NodeClaims/Nodes on the cluster so that we make sure that, if // LISTing instances takes a long time, our information is more updated by the time we get to nodeclaim and Node LIST // This works since our CloudProvider instances are deleted based on whether the NodeClaim exists or not, not vice-versa @@ -64,21 +65,20 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc if err != nil { return reconcile.Result{}, fmt.Errorf("listing cloudprovider VMs, %w", err) } - managedRetrieved := lo.Filter(retrieved, func(m *corev1beta1.NodeClaim, _ int) bool { - return m.DeletionTimestamp.IsZero() + managedRetrieved := lo.Filter(retrieved, func(nc *karpv1.NodeClaim, _ int) bool { + return nc.DeletionTimestamp.IsZero() }) - nodeClaims := &corev1beta1.NodeClaimList{} - if err := c.kubeClient.List(ctx, nodeClaims); err != nil { + nodeClaimList := &karpv1.NodeClaimList{} + if err = c.kubeClient.List(ctx, nodeClaimList); err != nil { return reconcile.Result{}, err } nodeList := &v1.NodeList{} if err := c.kubeClient.List(ctx, nodeList); err != nil { return reconcile.Result{}, err } - resolvedNodeClaims := lo.Filter(nodeClaims.Items, func(m corev1beta1.NodeClaim, _ int) bool { - return m.Status.ProviderID != "" - }) - resolvedProviderIDs := sets.New[string](lo.Map(resolvedNodeClaims, func(m corev1beta1.NodeClaim, _ int) string { return m.Status.ProviderID })...) + resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n karpv1.NodeClaim, _ int) (string, bool) { + return n.Status.ProviderID, n.Status.ProviderID != "" + })...) errs := make([]error, len(retrieved)) workqueue.ParallelizeUntil(ctx, 100, len(managedRetrieved), func(i int) { if !resolvedProviderIDs.Has(managedRetrieved[i].Status.ProviderID) && @@ -86,11 +86,14 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc errs[i] = c.garbageCollect(ctx, managedRetrieved[i], nodeList) } }) + if err = multierr.Combine(errs...); err != nil { + return reconcile.Result{}, err + } c.successfulCount++ - return reconcile.Result{RequeueAfter: lo.Ternary(c.successfulCount <= 20, time.Second*10, time.Minute*2)}, multierr.Combine(errs...) + return reconcile.Result{RequeueAfter: lo.Ternary(c.successfulCount <= 20, time.Second*10, time.Minute*2)}, nil } -func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, nodeList *v1.NodeList) error { +func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeList *v1.NodeList) error { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", nodeClaim.Status.ProviderID)) if err := c.cloudProvider.Delete(ctx, nodeClaim); err != nil { return corecloudprovider.IgnoreNodeClaimNotFoundError(err) @@ -109,6 +112,9 @@ func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *corev1beta1. return nil } -func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { - return controller.NewSingletonManagedBy(m) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim.garbagecollection"). + WatchesRawSource(singleton.Source()). + Complete(singleton.AsReconciler(c)) } diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index d88028651..b7e5fa349 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -22,6 +22,11 @@ import ( "testing" "time" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + "github.com/awslabs/operatorpkg/object" + opstatus "github.com/awslabs/operatorpkg/status" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" @@ -38,67 +43,65 @@ import ( "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/karpenter-provider-azure/pkg/test" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - "sigs.k8s.io/karpenter/pkg/operator/controller" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) var ctx context.Context -var stop context.CancelFunc var env *coretest.Environment var azureEnv *test.Environment var fakeClock *clock.FakeClock -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool var nodeClass *v1alpha2.AKSNodeClass var cluster *state.Cluster var cloudProvider *cloudprovider.CloudProvider -var garbageCollectionController controller.Controller -var coreProvisioner *provisioning.Provisioner +var garbageCollectionController *garbagecollection.Controller +var prov *provisioning.Provisioner func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) RegisterFailHandler(Fail) - RunSpecs(t, "NodeClaim") + RunSpecs(t, "GarbageCollection") } var _ = BeforeSuite(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) - - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx, stop = context.WithCancel(ctx) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) + // ctx, stop = context.WithCancel(ctx) azureEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) garbageCollectionController = garbagecollection.NewController(env.Client, cloudProvider) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + cluster = state.NewCluster(fakeClock, env.Client) + prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) var _ = AfterSuite(func() { - stop() + // stop() Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) var _ = BeforeEach(func() { nodeClass = test.AKSNodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClass.StatusConditions().SetTrue(opstatus.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, @@ -113,7 +116,10 @@ var _ = AfterEach(func() { ExpectCleanedUp(ctx, env.Client) }) -var _ = Describe("NodeClaimGarbageCollection", func() { +// TODO: move before/after each into the tests (see AWS) +// review tests themselves (very different from AWS?) +// (e.g. AWS has not a single ExpectPRovisioned? why?) +var _ = Describe("GarbageCollection", func() { var vm *armcompute.VirtualMachine var providerID string var err error @@ -122,7 +128,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { BeforeEach(func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{}) - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) vmName := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VMName @@ -130,6 +136,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { Expect(err).To(BeNil()) providerID = utils.ResourceIDToProviderID(ctx, *vm.ID) }) + It("should not delete an instance if it was not launched by a NodeClaim", func() { // Remove the "karpenter.sh/managed-by" tag (this isn't launched by a NodeClaim) vm.Properties = &armcompute.VirtualMachineProperties{ @@ -140,7 +147,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) @@ -150,7 +157,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { var vmName string for i := 0; i < 100; i++ { pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) if azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len() == 1 { vmName = azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VMName @@ -173,7 +180,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { ids = append(ids, *vm.ID) } } - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) wg := sync.WaitGroup{} for _, id := range ids { @@ -192,11 +199,11 @@ var _ = Describe("NodeClaimGarbageCollection", func() { It("should not delete all instances if they all have NodeClaim owners", func() { // Generate 100 instances that have different instanceIDs var ids []string - var nodeClaims []*corev1beta1.NodeClaim + var nodeClaims []*karpv1.NodeClaim var vmName string for i := 0; i < 100; i++ { pod := coretest.UnschedulablePod() - ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) if azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len() == 1 { vmName = azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VMName @@ -216,8 +223,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { instance.NodePoolTagKey: lo.ToPtr("default"), }, }) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: utils.ResourceIDToProviderID(ctx, *vm.ID), }, }) @@ -226,7 +233,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { nodeClaims = append(nodeClaims, nodeClaim) } } - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) wg := sync.WaitGroup{} for _, id := range ids { @@ -252,7 +259,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { } azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).NotTo(HaveOccurred()) }) @@ -263,8 +270,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { } azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: providerID, }, }) @@ -273,7 +280,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) ExpectApplied(ctx, env.Client, nodeClaim, node) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err := cloudProvider.Get(ctx, providerID) Expect(err).ToNot(HaveOccurred()) ExpectExists(ctx, env.Client, node) @@ -300,7 +307,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { } azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err = cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) @@ -316,7 +323,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) ExpectApplied(ctx, env.Client, node) - ExpectReconcileSucceeded(ctx, garbageCollectionController, client.ObjectKey{}) + ExpectSingletonReconciled(ctx, garbageCollectionController) _, err = cloudProvider.Get(ctx, providerID) Expect(err).To(HaveOccurred()) Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) diff --git a/pkg/controllers/nodeclaim/inplaceupdate/controller.go b/pkg/controllers/nodeclaim/inplaceupdate/controller.go index 557a4ab14..5ef2cdbc4 100644 --- a/pkg/controllers/nodeclaim/inplaceupdate/controller.go +++ b/pkg/controllers/nodeclaim/inplaceupdate/controller.go @@ -21,6 +21,7 @@ import ( "encoding/json" "fmt" + "github.com/awslabs/operatorpkg/reasonable" "github.com/samber/lo" "go.uber.org/zap/zapcore" "knative.dev/pkg/logging" @@ -31,8 +32,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + "sigs.k8s.io/karpenter/pkg/operator/injection" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" @@ -43,28 +45,22 @@ import ( type Controller struct { kubeClient client.Client - instanceProvider *instance.Provider + instanceProvider instance.Provider } -var _ corecontroller.TypedController[*v1beta1.NodeClaim] = &Controller{} - func NewController( kubeClient client.Client, - instanceProvider *instance.Provider, -) corecontroller.Controller { - controller := &Controller{ + instanceProvider instance.Provider, +) *Controller { + return &Controller{ kubeClient: kubeClient, instanceProvider: instanceProvider, } - - return corecontroller.Typed[*v1beta1.NodeClaim](kubeClient, controller) } -func (c *Controller) Name() string { - return "nodeclaim.inplaceupdate" -} +func (c *Controller) Reconcile(ctx context.Context, nodeClaim *karpv1.NodeClaim) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclaim.inplaceupdate") -func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim) (reconcile.Result, error) { if !nodeClaim.DeletionTimestamp.IsZero() { return reconcile.Result{}, nil } @@ -105,7 +101,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim return reconcile.Result{}, fmt.Errorf("getting azure VM for machine, %w", err) } - update := calculateVMPatch(options, vm) + update := CalculateVMPatch(options, vm) // This is safe only as long as we're not updating fields which we consider secret. // If we do/are, we need to redact them. logVMPatch(ctx, update) @@ -132,7 +128,7 @@ func (c *Controller) Reconcile(ctx context.Context, nodeClaim *v1beta1.NodeClaim return reconcile.Result{}, nil } -func calculateVMPatch( +func CalculateVMPatch( options *options.Options, // TODO: Can pass and consider NodeClaim and/or NodePool here if we need to in the future currentVM *armcompute.VirtualMachine, @@ -160,20 +156,25 @@ func calculateVMPatch( } } -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime.NewControllerManagedBy(m).For( - &v1beta1.NodeClaim{}, - builder.WithPredicates( - predicate.Or( - predicate.GenerationChangedPredicate{}, // Note that this will trigger on pod restart for all Machines. - ), - )).WithOptions(controller.Options{MaxConcurrentReconciles: 10}), - // TODO: Can add .Watches(&v1beta1.NodePool{}, nodeclaimutil.NodePoolEventHandler(c.kubeClient)) - // TODO: similar to https://github.com/kubernetes-sigs/karpenter/blob/main/pkg/controllers/nodeclaim/disruption/controller.go#L214C3-L217C5 - // TODO: if/when we need to monitor provisoner changes and flow updates on the NodePool down to the underlying VMs. - ) +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim.inplaceupdate"). + For( + &karpv1.NodeClaim{}, + builder.WithPredicates( + predicate.Or( + predicate.GenerationChangedPredicate{}, // Note that this will trigger on pod restart for all Machines. + ), + )). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + // TODO: Can add .Watches(&karpv1.NodePool{}, nodeclaimutil.NodePoolEventHandler(c.kubeClient)) + // TODO: similar to https://github.com/kubernetes-sigs/karpenter/blob/main/pkg/controllers/nodeclaim/disruption/controller.go#L214C3-L217C5 + // TODO: if/when we need to monitor provisoner changes and flow updates on the NodePool down to the underlying VMs. + Complete(reconcile.AsReconciler(m.GetClient(), c)) } - func logVMPatch(ctx context.Context, update *armcompute.VirtualMachineUpdate) { if logging.FromContext(ctx).Level().Enabled(zapcore.DebugLevel) { rawStr := "" diff --git a/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go b/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go index 93f1e6606..03d0f4af4 100644 --- a/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go +++ b/pkg/controllers/nodeclaim/inplaceupdate/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package inplaceupdate +package inplaceupdate_test import ( "context" @@ -25,26 +25,24 @@ import ( . "github.com/onsi/gomega" "github.com/samber/lo" . "knative.dev/pkg/logging/testing" - "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/inplaceupdate" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/Azure/karpenter-provider-azure/pkg/test" "github.com/Azure/karpenter-provider-azure/pkg/utils" ) var ctx context.Context -var stop context.CancelFunc var env *coretest.Environment var azureEnv *test.Environment -var inPlaceUpdateController corecontroller.Controller +var inPlaceUpdateController *inplaceupdate.Controller func TestInPlaceUpdate(t *testing.T) { ctx = TestContextWithLogger(t) @@ -54,17 +52,14 @@ func TestInPlaceUpdate(t *testing.T) { var _ = BeforeSuite(func() { ctx = coreoptions.ToContext(ctx, coretest.Options()) - - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - - ctx, stop = context.WithCancel(ctx) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) + // ctx, stop = context.WithCancel(ctx) azureEnv = test.NewEnvironment(ctx, env) - - inPlaceUpdateController = NewController(env.Client, azureEnv.InstanceProvider) + inPlaceUpdateController = inplaceupdate.NewController(env.Client, azureEnv.InstanceProvider) }) var _ = AfterSuite(func() { - stop() + //stop() Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) @@ -101,13 +96,13 @@ var _ = Describe("Unit tests", func() { }, } - hash1, err := HashFromVM(vm1) + hash1, err := inplaceupdate.HashFromVM(vm1) Expect(err).ToNot(HaveOccurred()) - hash2, err := HashFromVM(vm2) + hash2, err := inplaceupdate.HashFromVM(vm2) Expect(err).ToNot(HaveOccurred()) - hash3, err := HashFromVM(vm3) + hash3, err := inplaceupdate.HashFromVM(vm3) Expect(err).ToNot(HaveOccurred()) Expect(hash1).To(Equal(hash2)) @@ -124,7 +119,7 @@ var _ = Describe("Unit tests", func() { "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3", } - hash1, err := HashFromNodeClaim(options, nil) + hash1, err := inplaceupdate.HashFromNodeClaim(options, nil) Expect(err).ToNot(HaveOccurred()) options.NodeIdentities = []string{ @@ -132,7 +127,7 @@ var _ = Describe("Unit tests", func() { "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid3", } - hash2, err := HashFromNodeClaim(options, nil) + hash2, err := inplaceupdate.HashFromNodeClaim(options, nil) Expect(err).ToNot(HaveOccurred()) options.NodeIdentities = []string{ @@ -140,7 +135,7 @@ var _ = Describe("Unit tests", func() { "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", } - hash3, err := HashFromNodeClaim(options, nil) + hash3, err := inplaceupdate.HashFromNodeClaim(options, nil) Expect(err).ToNot(HaveOccurred()) Expect(hash1).To(Equal(hash2)) @@ -156,7 +151,7 @@ var _ = Describe("Unit tests", func() { options.NodeIdentities = []string{ "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", } - update := calculateVMPatch(options, currentVM) + update := inplaceupdate.CalculateVMPatch(options, currentVM) Expect(update).ToNot(BeNil()) Expect(update.Identity).ToNot(BeNil()) @@ -177,7 +172,7 @@ var _ = Describe("Unit tests", func() { options.NodeIdentities = []string{ "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", } - update := calculateVMPatch(options, currentVM) + update := inplaceupdate.CalculateVMPatch(options, currentVM) Expect(update).ToNot(BeNil()) Expect(update.Identity).ToNot(BeNil()) @@ -200,7 +195,7 @@ var _ = Describe("Unit tests", func() { "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid2", "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", } - update := calculateVMPatch(options, currentVM) + update := inplaceupdate.CalculateVMPatch(options, currentVM) Expect(update).To(BeNil()) }) @@ -219,7 +214,7 @@ var _ = Describe("Unit tests", func() { options.NodeIdentities = []string{ "/subscriptions/1234/resourceGroups/mcrg/providers/Microsoft.ManagedIdentity/userAssignedIdentities/myid1", } - update := calculateVMPatch(options, currentVM) + update := inplaceupdate.CalculateVMPatch(options, currentVM) Expect(update).To(BeNil()) }) @@ -229,7 +224,7 @@ var _ = Describe("Unit tests", func() { var _ = Describe("In Place Update Controller", func() { var vmName string var vm *armcompute.VirtualMachine - var nodeClaim *corev1beta1.NodeClaim + var nodeClaim *karpv1.NodeClaim BeforeEach(func() { vmName = "vm-a" @@ -238,8 +233,8 @@ var _ = Describe("In Place Update Controller", func() { Name: lo.ToPtr(vmName), } - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ - Status: corev1beta1.NodeClaimStatus{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ + Status: karpv1.NodeClaimStatus{ ProviderID: utils.ResourceIDToProviderID(ctx, *vm.ID), }, }) @@ -256,14 +251,14 @@ var _ = Describe("In Place Update Controller", func() { Context("Basic tests", func() { It("should not call Azure if the hash matches", func() { azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) - hash, err := HashFromNodeClaim(options.FromContext(ctx), nodeClaim) + hash, err := inplaceupdate.HashFromNodeClaim(options.FromContext(ctx), nodeClaim) Expect(err).ToNot(HaveOccurred()) // Force the goal hash into annotations here, which should prevent the reconciler from doing anything on Azure nodeClaim.Annotations = map[string]string{v1alpha2.AnnotationInPlaceUpdateHash: hash} ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, inPlaceUpdateController, nodeClaim) Expect(azureEnv.VirtualMachinesAPI.VirtualMachineUpdateBehavior.Calls()).To(Equal(0)) @@ -278,7 +273,7 @@ var _ = Describe("In Place Update Controller", func() { azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, inPlaceUpdateController, nodeClaim) updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) Expect(err).ToNot(HaveOccurred()) @@ -304,7 +299,7 @@ var _ = Describe("In Place Update Controller", func() { })) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, inPlaceUpdateController, nodeClaim) updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) Expect(err).ToNot(HaveOccurred()) @@ -327,7 +322,7 @@ var _ = Describe("In Place Update Controller", func() { azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, inPlaceUpdateController, nodeClaim) Expect(azureEnv.VirtualMachinesAPI.VirtualMachineUpdateBehavior.Calls()).To(Equal(0)) @@ -355,7 +350,7 @@ var _ = Describe("In Place Update Controller", func() { azureEnv.VirtualMachinesAPI.Instances.Store(lo.FromPtr(vm.ID), *vm) ExpectApplied(ctx, env.Client, nodeClaim) - ExpectReconcileSucceeded(ctx, inPlaceUpdateController, client.ObjectKeyFromObject(nodeClaim)) + ExpectObjectReconciled(ctx, env.Client, inPlaceUpdateController, nodeClaim) updatedVM, err := azureEnv.InstanceProvider.Get(ctx, vmName) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/controllers/nodeclaim/inplaceupdate/utils.go b/pkg/controllers/nodeclaim/inplaceupdate/utils.go index b64551e09..b0cd4f3bf 100644 --- a/pkg/controllers/nodeclaim/inplaceupdate/utils.go +++ b/pkg/controllers/nodeclaim/inplaceupdate/utils.go @@ -22,7 +22,7 @@ import ( "strconv" "k8s.io/apimachinery/pkg/util/sets" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" @@ -64,7 +64,7 @@ func HashFromVM(vm *armcompute.VirtualMachine) (string, error) { } // HashFromNodeClaim calculates an inplace update hash from the specified machine and options -func HashFromNodeClaim(options *options.Options, _ *v1beta1.NodeClaim) (string, error) { +func HashFromNodeClaim(options *options.Options, _ *karpv1.NodeClaim) (string, error) { hashStruct := &inPlaceUpdateFields{ Identities: sets.New(options.NodeIdentities...), } From 785c4e352ff620699e1310146a273ac9d991f8c4 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:57:19 +0000 Subject: [PATCH 15/47] chore: add nodeclass status controller --- .../nodeclass/status/controller.go | 98 +++++++++++++++++++ pkg/controllers/nodeclass/status/readiness.go | 34 +++++++ 2 files changed, 132 insertions(+) create mode 100644 pkg/controllers/nodeclass/status/controller.go create mode 100644 pkg/controllers/nodeclass/status/readiness.go diff --git a/pkg/controllers/nodeclass/status/controller.go b/pkg/controllers/nodeclass/status/controller.go new file mode 100644 index 000000000..8eb62ded6 --- /dev/null +++ b/pkg/controllers/nodeclass/status/controller.go @@ -0,0 +1,98 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status + +import ( + "context" + + "go.uber.org/multierr" + "k8s.io/apimachinery/pkg/api/equality" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" + + "sigs.k8s.io/karpenter/pkg/utils/result" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/awslabs/operatorpkg/reasonable" +) + +type nodeClassStatusReconciler interface { + Reconcile(context.Context, *v1alpha2.AKSNodeClass) (reconcile.Result, error) +} + +type Controller struct { + kubeClient client.Client + + readiness *Readiness //TODO : Remove this when we have sub status conditions +} + +func NewController(kubeClient client.Client) *Controller { + return &Controller{ + kubeClient: kubeClient, + + readiness: &Readiness{}, + } +} + +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.status") + + if !controllerutil.ContainsFinalizer(nodeClass, v1alpha2.TerminationFinalizer) { + stored := nodeClass.DeepCopy() + controllerutil.AddFinalizer(nodeClass, v1alpha2.TerminationFinalizer) + if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { + return reconcile.Result{}, err + } + } + stored := nodeClass.DeepCopy() + + var results []reconcile.Result + var errs error + for _, reconciler := range []nodeClassStatusReconciler{ + c.readiness, + } { + res, err := reconciler.Reconcile(ctx, nodeClass) + errs = multierr.Append(errs, err) + results = append(results, res) + } + + if !equality.Semantic.DeepEqual(stored, nodeClass) { + if err := c.kubeClient.Status().Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { + errs = multierr.Append(errs, client.IgnoreNotFound(err)) + } + } + if errs != nil { + return reconcile.Result{}, errs + } + return result.Min(results...), nil +} + +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.status"). + For(&v1alpha2.AKSNodeClass{}). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) +} diff --git a/pkg/controllers/nodeclass/status/readiness.go b/pkg/controllers/nodeclass/status/readiness.go new file mode 100644 index 000000000..51136fc06 --- /dev/null +++ b/pkg/controllers/nodeclass/status/readiness.go @@ -0,0 +1,34 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package status + +import ( + "context" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/awslabs/operatorpkg/status" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type Readiness struct { +} + +func (n Readiness) Reconcile(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) (reconcile.Result, error) { + nodeClass.StatusConditions().SetTrue(status.ConditionReady) + return reconcile.Result{}, nil +} From 6a2660f66d2b087cd78ef89423eb4b653433c3cb Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 07:58:14 +0000 Subject: [PATCH 16/47] chore: migrate providers to v1 API --- pkg/providers/imagefamily/azlinux.go | 10 +- .../imagefamily/bootstrap/aksbootstrap.go | 4 +- .../imagefamily/bootstrap/bootstrap.go | 4 +- pkg/providers/imagefamily/image.go | 2 +- pkg/providers/imagefamily/resolver.go | 32 ++- pkg/providers/imagefamily/ubuntu_2204.go | 10 +- pkg/providers/instance/instance.go | 117 ++++++----- pkg/providers/instance/instance_test.go | 34 ++-- pkg/providers/instance/suite_test.go | 42 ++-- pkg/providers/instancetype/instancetype.go | 96 +++++---- pkg/providers/instancetype/instancetypes.go | 120 ++++++++--- pkg/providers/instancetype/suite_test.go | 190 ++++++++---------- .../launchtemplate/launchtemplate.go | 10 +- 13 files changed, 380 insertions(+), 291 deletions(-) diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index 3d36d8cc2..771c15e1e 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" ) @@ -49,7 +49,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { CommunityImage: AzureLinuxGen2CommunityImage, PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), }, @@ -57,7 +57,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { CommunityImage: AzureLinuxGen1CommunityImage, PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), ), }, @@ -65,7 +65,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { CommunityImage: AzureLinuxGen2ArmCommunityImage, PublicGalleryURL: AKSAzureLinuxPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), }, @@ -73,7 +73,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { } // UserData returns the default userdata script for the image Family -func (u AzureLinux) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u AzureLinux) UserData(kubeletConfig *v1alpha2.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go index e337e4188..a1fbbf566 100644 --- a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go @@ -24,12 +24,12 @@ import ( "strings" "text/template" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/blang/semver/v4" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -562,7 +562,7 @@ func normalizeResourceGroupNameForLabel(resourceGroupName string) string { return truncated } -func KubeletConfigToMap(kubeletConfig *corev1beta1.KubeletConfiguration) map[string]string { +func KubeletConfigToMap(kubeletConfig *v1alpha2.KubeletConfiguration) map[string]string { args := make(map[string]string) if kubeletConfig == nil { diff --git a/pkg/providers/imagefamily/bootstrap/bootstrap.go b/pkg/providers/imagefamily/bootstrap/bootstrap.go index 2bf1fab8d..f23468a4b 100644 --- a/pkg/providers/imagefamily/bootstrap/bootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/bootstrap.go @@ -17,15 +17,15 @@ limitations under the License. package bootstrap import ( + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" core "k8s.io/api/core/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) // Options is the node bootstrapping parameters passed from Karpenter to the provisioning node type Options struct { ClusterName string ClusterEndpoint string - KubeletConfig *corev1beta1.KubeletConfiguration + KubeletConfig *v1alpha2.KubeletConfiguration Taints []core.Taint `hash:"set"` Labels map[string]string `hash:"set"` CABundle *string diff --git a/pkg/providers/imagefamily/image.go b/pkg/providers/imagefamily/image.go index ff5a7e70c..af336aabe 100644 --- a/pkg/providers/imagefamily/image.go +++ b/pkg/providers/imagefamily/image.go @@ -66,7 +66,7 @@ func NewProvider(kubernetesInterface kubernetes.Interface, kubernetesVersionCach func (p *Provider) Get(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, instanceType *cloudprovider.InstanceType, imageFamily ImageFamily) (string, error) { defaultImages := imageFamily.DefaultImages() for _, defaultImage := range defaultImages { - if err := instanceType.Requirements.Compatible(defaultImage.Requirements, v1alpha2.AllowUndefinedLabels); err == nil { + if err := instanceType.Requirements.Compatible(defaultImage.Requirements, v1alpha2.AllowUndefinedWellKnownAndRestrictedLabels); err == nil { communityImageName, publicGalleryURL := defaultImage.CommunityImage, defaultImage.PublicGalleryURL return p.GetImageID(ctx, communityImageName, publicGalleryURL, nodeClass.Spec.GetImageVersion()) } diff --git a/pkg/providers/imagefamily/resolver.go b/pkg/providers/imagefamily/resolver.go index 787905c9a..f11672c3d 100644 --- a/pkg/providers/imagefamily/resolver.go +++ b/pkg/providers/imagefamily/resolver.go @@ -19,7 +19,7 @@ package imagefamily import ( "context" - core "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" @@ -28,10 +28,10 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/instancetype" template "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" + "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/samber/lo" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" - "sigs.k8s.io/karpenter/pkg/utils/resources" ) const ( @@ -54,8 +54,8 @@ type Resolver struct { // ImageFamily can be implemented to override the default logic for generating dynamic launch template parameters type ImageFamily interface { UserData( - kubeletConfig *corev1beta1.KubeletConfiguration, - taints []core.Taint, + kubeletConfig *v1alpha2.KubeletConfiguration, + taints []corev1.Taint, labels map[string]string, caBundle *string, instanceType *cloudprovider.InstanceType, @@ -75,7 +75,7 @@ func New(_ client.Client, imageProvider *Provider) *Resolver { } // Resolve fills in dynamic launch template parameters -func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceType *cloudprovider.InstanceType, +func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceType *cloudprovider.InstanceType, staticParameters *template.StaticParameters) (*template.Parameters, error) { imageFamily := getImageFamily(nodeClass.Spec.ImageFamily, staticParameters) imageID, err := r.imageProvider.Get(ctx, nodeClass, instanceType, imageFamily) @@ -84,14 +84,24 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, return nil, err } - kubeletConfig := nodeClaim.Spec.Kubelet + kubeletConfig := nodeClass.Spec.Kubelet if kubeletConfig == nil { - kubeletConfig = &corev1beta1.KubeletConfiguration{} + kubeletConfig = &v1alpha2.KubeletConfiguration{} + } + + taints := lo.Flatten([][]corev1.Taint{ + nodeClaim.Spec.Taints, + nodeClaim.Spec.StartupTaints, + }) + if _, found := lo.Find(taints, func(t corev1.Taint) bool { + return t.MatchTaint(&karpv1.UnregisteredNoExecuteTaint) + }); !found { + taints = append(taints, karpv1.UnregisteredNoExecuteTaint) } // TODO: revisit computeResources and maxPods implementation - kubeletConfig.KubeReserved = resources.StringMap(instanceType.Overhead.KubeReserved) - kubeletConfig.SystemReserved = resources.StringMap(instanceType.Overhead.SystemReserved) + kubeletConfig.KubeReserved = utils.StringMap(instanceType.Overhead.KubeReserved) + kubeletConfig.SystemReserved = utils.StringMap(instanceType.Overhead.SystemReserved) kubeletConfig.EvictionHard = map[string]string{ instancetype.MemoryAvailable: instanceType.Overhead.EvictionThreshold.Memory().String()} kubeletConfig.MaxPods = lo.ToPtr(getMaxPods(staticParameters.NetworkPlugin)) @@ -100,7 +110,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, StaticParameters: staticParameters, UserData: imageFamily.UserData( kubeletConfig, - append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...), + taints, staticParameters.Labels, staticParameters.CABundle, instanceType, diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index a7c3b8ee2..54f7b2bbe 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -23,7 +23,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily/bootstrap" "github.com/Azure/karpenter-provider-azure/pkg/providers/launchtemplate/parameters" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" ) @@ -49,7 +49,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { CommunityImage: Ubuntu2204Gen2CommunityImage, PublicGalleryURL: AKSUbuntuPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), }, @@ -57,7 +57,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { CommunityImage: Ubuntu2204Gen1CommunityImage, PublicGalleryURL: AKSUbuntuPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureAmd64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV1), ), }, @@ -65,7 +65,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { CommunityImage: Ubuntu2204Gen2ArmCommunityImage, PublicGalleryURL: AKSUbuntuPublicGalleryURL, Requirements: scheduling.NewRequirements( - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64), scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpIn, v1alpha2.HyperVGenerationV2), ), }, @@ -73,7 +73,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { } // UserData returns the default userdata script for the image Family -func (u Ubuntu2204) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u Ubuntu2204) UserData(kubeletConfig *v1alpha2.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 4c2ea355f..e3ca129f6 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -44,7 +44,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" //nolint SA1019 - deprecated package "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2022-08-01/compute" @@ -56,16 +56,16 @@ import ( ) var ( - NodePoolTagKey = strings.ReplaceAll(corev1beta1.NodePoolLabelKey, "/", "_") + NodePoolTagKey = strings.ReplaceAll(karpv1.NodePoolLabelKey, "/", "_") listQuery string CapacityTypeToPriority = map[string]string{ - corev1beta1.CapacityTypeSpot: string(compute.Spot), - corev1beta1.CapacityTypeOnDemand: string(compute.Regular), + karpv1.CapacityTypeSpot: string(compute.Spot), + karpv1.CapacityTypeOnDemand: string(compute.Regular), } PriorityToCapacityType = map[string]string{ - string(compute.Spot): corev1beta1.CapacityTypeSpot, - string(compute.Regular): corev1beta1.CapacityTypeOnDemand, + string(compute.Spot): karpv1.CapacityTypeSpot, + string(compute.Regular): karpv1.CapacityTypeOnDemand, } SubscriptionQuotaReachedReason = "SubscriptionQuotaReached" @@ -79,10 +79,22 @@ var ( type Resource = map[string]interface{} -type Provider struct { +type Provider interface { + Create(context.Context, *v1alpha2.AKSNodeClass, *karpv1.NodeClaim, []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, error) + Get(context.Context, string) (*armcompute.VirtualMachine, error) + List(context.Context) ([]*armcompute.VirtualMachine, error) + Delete(context.Context, string) error + // CreateTags(context.Context, string, map[string]string) error + Update(context.Context, string, armcompute.VirtualMachineUpdate) error +} + +// assert that DefaultProvider implements Provider interface +var _ Provider = (*DefaultProvider)(nil) + +type DefaultProvider struct { location string azClient *AZClient - instanceTypeProvider *instancetype.Provider + instanceTypeProvider instancetype.Provider launchTemplateProvider *launchtemplate.Provider loadBalancerProvider *loadbalancer.Provider resourceGroup string @@ -91,9 +103,9 @@ type Provider struct { unavailableOfferings *cache.UnavailableOfferings } -func NewProvider( +func NewDefaultProvider( azClient *AZClient, - instanceTypeProvider *instancetype.Provider, + instanceTypeProvider instancetype.Provider, launchTemplateProvider *launchtemplate.Provider, loadBalancerProvider *loadbalancer.Provider, offeringsCache *cache.UnavailableOfferings, @@ -101,9 +113,9 @@ func NewProvider( resourceGroup string, subnetID string, subscriptionID string, -) *Provider { +) *DefaultProvider { listQuery = GetListQueryBuilder(resourceGroup).String() - return &Provider{ + return &DefaultProvider{ azClient: azClient, instanceTypeProvider: instanceTypeProvider, launchTemplateProvider: launchTemplateProvider, @@ -118,7 +130,7 @@ func NewProvider( // Create an instance given the constraints. // instanceTypes should be sorted by priority for spot capacity type. -func (p *Provider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, error) { +func (p *DefaultProvider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, error) { instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...)) vm, instanceType, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes) if err != nil { @@ -141,11 +153,11 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, return vm, nil } -func (p *Provider) Update(ctx context.Context, vmName string, update armcompute.VirtualMachineUpdate) error { +func (p *DefaultProvider) Update(ctx context.Context, vmName string, update armcompute.VirtualMachineUpdate) error { return UpdateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, update) } -func (p *Provider) Get(ctx context.Context, vmName string) (*armcompute.VirtualMachine, error) { +func (p *DefaultProvider) Get(ctx context.Context, vmName string) (*armcompute.VirtualMachine, error) { var vm armcompute.VirtualMachinesClientGetResponse var err error @@ -159,7 +171,7 @@ func (p *Provider) Get(ctx context.Context, vmName string) (*armcompute.VirtualM return &vm.VirtualMachine, nil } -func (p *Provider) List(ctx context.Context) ([]*armcompute.VirtualMachine, error) { +func (p *DefaultProvider) List(ctx context.Context) ([]*armcompute.VirtualMachine, error) { req := NewQueryRequest(&(p.subscriptionID), listQuery) client := p.azClient.azureResourceGraphClient data, err := GetResourceData(ctx, client, *req) @@ -177,13 +189,13 @@ func (p *Provider) List(ctx context.Context) ([]*armcompute.VirtualMachine, erro return vmList, nil } -func (p *Provider) Delete(ctx context.Context, resourceName string) error { +func (p *DefaultProvider) Delete(ctx context.Context, resourceName string) error { logging.FromContext(ctx).Debugf("Deleting virtual machine %s and associated resources", resourceName) return p.cleanupAzureResources(ctx, resourceName) } // createAKSIdentifyingExtension attaches a VM extension to identify that this VM participates in an AKS cluster -func (p *Provider) createAKSIdentifyingExtension(ctx context.Context, vmName string) (err error) { +func (p *DefaultProvider) createAKSIdentifyingExtension(ctx context.Context, vmName string) (err error) { vmExt := p.getAKSIdentifyingExtension() vmExtName := *vmExt.Name logging.FromContext(ctx).Debugf("Creating virtual machine AKS identifying extension for %s", vmName) @@ -196,10 +208,9 @@ func (p *Provider) createAKSIdentifyingExtension(ctx context.Context, vmName str return nil } -func (p *Provider) newNetworkInterfaceForVM(vmName string, backendPools *loadbalancer.BackendAddressPools, instanceType *corecloudprovider.InstanceType) armnetwork.Interface { +func (p *DefaultProvider) newNetworkInterfaceForVM(vmName string, backendPools *loadbalancer.BackendAddressPools, instanceType *corecloudprovider.InstanceType) armnetwork.Interface { var ipv4BackendPools []*armnetwork.BackendAddressPool for _, poolID := range backendPools.IPv4PoolIDs { - poolID := poolID ipv4BackendPools = append(ipv4BackendPools, &armnetwork.BackendAddressPool{ ID: &poolID, }) @@ -238,7 +249,7 @@ func GenerateResourceName(nodeClaimName string) string { return fmt.Sprintf("aks-%s", nodeClaimName) } -func (p *Provider) createNetworkInterface(ctx context.Context, nicName string, launchTemplateConfig *launchtemplate.Template, instanceType *corecloudprovider.InstanceType) (string, error) { +func (p *DefaultProvider) createNetworkInterface(ctx context.Context, nicName string, launchTemplateConfig *launchtemplate.Template, instanceType *corecloudprovider.InstanceType) (string, error) { backendPools, err := p.loadBalancerProvider.LoadBalancerBackendPools(ctx) if err != nil { return "", err @@ -345,7 +356,7 @@ func setVMPropertiesStorageProfile(vmProperties *armcompute.VirtualMachineProper // setVMPropertiesBillingProfile sets a default MaxPrice of -1 for Spot func setVMPropertiesBillingProfile(vmProperties *armcompute.VirtualMachineProperties, capacityType string) { - if capacityType == corev1beta1.CapacityTypeSpot { + if capacityType == karpv1.CapacityTypeSpot { vmProperties.EvictionPolicy = to.Ptr(armcompute.VirtualMachineEvictionPolicyTypesDelete) vmProperties.BillingProfile = &armcompute.BillingProfile{ MaxPrice: to.Ptr(float64(-1)), @@ -354,13 +365,13 @@ func setVMPropertiesBillingProfile(vmProperties *armcompute.VirtualMachineProper } // setNodePoolNameTag sets "karpenter.sh/nodepool" tag -func setNodePoolNameTag(tags map[string]*string, nodeClaim *corev1beta1.NodeClaim) { - if val, ok := nodeClaim.Labels[corev1beta1.NodePoolLabelKey]; ok { +func setNodePoolNameTag(tags map[string]*string, nodeClaim *karpv1.NodeClaim) { + if val, ok := nodeClaim.Labels[karpv1.NodePoolLabelKey]; ok { tags[NodePoolTagKey] = &val } } -func (p *Provider) createVirtualMachine(ctx context.Context, vm armcompute.VirtualMachine, vmName string) (*armcompute.VirtualMachine, error) { +func (p *DefaultProvider) createVirtualMachine(ctx context.Context, vm armcompute.VirtualMachine, vmName string) (*armcompute.VirtualMachine, error) { result, err := CreateVirtualMachine(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, vmName, vm) if err != nil { logging.FromContext(ctx).Errorf("Creating virtual machine %q failed: %v", vmName, err) @@ -370,8 +381,8 @@ func (p *Provider) createVirtualMachine(ctx context.Context, vm armcompute.Virtu return result, nil } -func (p *Provider) launchInstance( - ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, *corecloudprovider.InstanceType, error) { +func (p *DefaultProvider) launchInstance( + ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*armcompute.VirtualMachine, *corecloudprovider.InstanceType, error) { instanceType, capacityType, zone := p.pickSkuSizePriorityAndZone(ctx, nodeClaim, instanceTypes) if instanceType == nil { return nil, nil, corecloudprovider.NewInsufficientCapacityError(fmt.Errorf("no instance types available")) @@ -413,7 +424,7 @@ func (p *Provider) launchInstance( } // nolint:gocyclo -func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corecloudprovider.InstanceType, zone, capacityType string, err error) error { +func (p *DefaultProvider) handleResponseErrors(ctx context.Context, instanceType *corecloudprovider.InstanceType, zone, capacityType string, err error) error { if sdkerrors.LowPriorityQuotaHasBeenReached(err) { // Mark in cache that spot quota has been reached for this subscription p.unavailableOfferings.MarkSpotUnavailableWithTTL(ctx, SubscriptionQuotaReachedTTL) @@ -427,15 +438,15 @@ func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corec logging.FromContext(ctx).Error(err) for _, offering := range instanceType.Offerings { - if offering.CapacityType != capacityType { + if getOfferingCapacityType(offering) != capacityType { continue } // If we have a quota limit of 0 vcpus, we mark the offerings unavailable for an hour. // CPU limits of 0 are usually due to a subscription having no allocated quota for that instance type at all on the subscription. if cpuLimitIsZero(err) { - p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SubscriptionQuotaReachedReason, instanceType.Name, offering.Zone, capacityType, SubscriptionQuotaReachedTTL) + p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SubscriptionQuotaReachedReason, instanceType.Name, getOfferingZone(offering), capacityType, SubscriptionQuotaReachedTTL) } else { - p.unavailableOfferings.MarkUnavailable(ctx, SubscriptionQuotaReachedReason, instanceType.Name, offering.Zone, capacityType) + p.unavailableOfferings.MarkUnavailable(ctx, SubscriptionQuotaReachedReason, instanceType.Name, getOfferingZone(offering), capacityType) } } return fmt.Errorf("subscription level %s vCPU quota for %s has been reached (may try provision an alternative instance type)", capacityType, instanceType.Name) @@ -447,16 +458,16 @@ func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corec // - zonal restrictions are filtered out internally by sku.AvailabilityZones, and don't get offerings skuNotAvailableTTL := SKUNotAvailableSpotTTL err = fmt.Errorf("out of spot capacity for %s: %w", instanceType.Name, err) - if capacityType == corev1beta1.CapacityTypeOnDemand { // should not happen, defensive check + if capacityType == karpv1.CapacityTypeOnDemand { // should not happen, defensive check err = fmt.Errorf("unexpected SkuNotAvailable error for %s (on-demand): %w", instanceType.Name, err) skuNotAvailableTTL = SKUNotAvailableOnDemandTTL // still mark all offerings as unavailable, but with a longer TTL } // mark the instance type as unavailable for all offerings/zones for the capacity type for _, offering := range instanceType.Offerings { - if offering.CapacityType != capacityType { + if getOfferingCapacityType(offering) != capacityType { continue } - p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SKUNotAvailableReason, instanceType.Name, offering.Zone, capacityType, skuNotAvailableTTL) + p.unavailableOfferings.MarkUnavailableWithTTL(ctx, SKUNotAvailableReason, instanceType.Name, getOfferingZone(offering), capacityType, skuNotAvailableTTL) } logging.FromContext(ctx).Error(err) @@ -464,8 +475,8 @@ func (p *Provider) handleResponseErrors(ctx context.Context, instanceType *corec } if sdkerrors.ZonalAllocationFailureOccurred(err) { logging.FromContext(ctx).With("zone", zone).Error(err) - p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, corev1beta1.CapacityTypeOnDemand) - p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, corev1beta1.CapacityTypeSpot) + p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, karpv1.CapacityTypeOnDemand) + p.unavailableOfferings.MarkUnavailable(ctx, ZonalAllocationFailureReason, instanceType.Name, zone, karpv1.CapacityTypeSpot) return fmt.Errorf("unable to allocate resources in the selected zone (%s). (will try a different zone to fulfill your request)", zone) } @@ -494,14 +505,14 @@ func cpuLimitIsZero(err error) bool { return strings.Contains(err.Error(), "Current Limit: 0") } -func (p *Provider) applyTemplateToNic(nic *armnetwork.Interface, template *launchtemplate.Template) { +func (p *DefaultProvider) applyTemplateToNic(nic *armnetwork.Interface, template *launchtemplate.Template) { // set tags nic.Tags = template.Tags } -func (p *Provider) getLaunchTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, +func (p *DefaultProvider) getLaunchTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceType *corecloudprovider.InstanceType, capacityType string) (*launchtemplate.Template, error) { - additionalLabels := lo.Assign(GetAllSingleValuedRequirementLabels(instanceType), map[string]string{corev1beta1.CapacityTypeLabelKey: capacityType}) + additionalLabels := lo.Assign(GetAllSingleValuedRequirementLabels(instanceType), map[string]string{karpv1.CapacityTypeLabelKey: capacityType}) launchTemplate, err := p.launchTemplateProvider.GetTemplate(ctx, nodeClass, nodeClaim, instanceType, additionalLabels) if err != nil { @@ -528,7 +539,7 @@ func GetAllSingleValuedRequirementLabels(instanceType *corecloudprovider.Instanc } // pick the "best" SKU, priority and zone, from InstanceType options (and their offerings) in the request -func (p *Provider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*corecloudprovider.InstanceType, string, string) { +func (p *DefaultProvider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *karpv1.NodeClaim, instanceTypes []*corecloudprovider.InstanceType) (*corecloudprovider.InstanceType, string, string) { if len(instanceTypes) == 0 { return nil, "", "" } @@ -540,9 +551,9 @@ func (p *Provider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *co // Zone - ideally random/spread from requested zones that support given Priority requestedZones := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone) priorityOfferings := lo.Filter(instanceType.Offerings.Available(), func(o corecloudprovider.Offering, _ int) bool { - return o.CapacityType == priority && requestedZones.Has(o.Zone) + return getOfferingCapacityType(o) == priority && requestedZones.Has(getOfferingZone(o)) }) - zonesWithPriority := lo.Map(priorityOfferings, func(o corecloudprovider.Offering, _ int) string { return o.Zone }) + zonesWithPriority := lo.Map(priorityOfferings, func(o corecloudprovider.Offering, _ int) string { return getOfferingZone(o) }) if zone, ok := sets.New(zonesWithPriority...).PopAny(); ok { if len(zone) > 0 { // Zones in zonal Offerings have - format; the zone returned from here will be used for VM instantiation, @@ -554,7 +565,7 @@ func (p *Provider) pickSkuSizePriorityAndZone(ctx context.Context, nodeClaim *co return nil, "", "" } -func (p *Provider) cleanupAzureResources(ctx context.Context, resourceName string) (err error) { +func (p *DefaultProvider) cleanupAzureResources(ctx context.Context, resourceName string) (err error) { vmErr := deleteVirtualMachineIfExists(ctx, p.azClient.virtualMachinesClient, p.resourceGroup, resourceName) if vmErr != nil { logging.FromContext(ctx).Errorf("virtualMachine.Delete for %s failed: %v", resourceName, vmErr) @@ -575,17 +586,17 @@ func (p *Provider) cleanupAzureResources(ctx context.Context, resourceName strin // // This returns from a single pre-selected InstanceType, rather than all InstanceType options in nodeRequest, // because Azure Cloud Provider does client-side selection of particular InstanceType from options -func (p *Provider) getPriorityForInstanceType(nodeClaim *corev1beta1.NodeClaim, instanceType *corecloudprovider.InstanceType) string { +func (p *DefaultProvider) getPriorityForInstanceType(nodeClaim *karpv1.NodeClaim, instanceType *corecloudprovider.InstanceType) string { requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) - if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { + if requirements.Get(karpv1.CapacityTypeLabelKey).Has(karpv1.CapacityTypeSpot) { for _, offering := range instanceType.Offerings.Available() { - if requirements.Get(v1.LabelTopologyZone).Has(offering.Zone) && offering.CapacityType == corev1beta1.CapacityTypeSpot { - return corev1beta1.CapacityTypeSpot + if requirements.Get(v1.LabelTopologyZone).Has(getOfferingZone(offering)) && getOfferingCapacityType(offering) == karpv1.CapacityTypeSpot { + return karpv1.CapacityTypeSpot } } } - return corev1beta1.CapacityTypeOnDemand + return karpv1.CapacityTypeOnDemand } func orderInstanceTypesByPrice(instanceTypes []*corecloudprovider.InstanceType, requirements scheduling.Requirements) []*corecloudprovider.InstanceType { @@ -614,7 +625,7 @@ func GetCapacityType(instance *armcompute.VirtualMachine) string { return "" } -func (p *Provider) getAKSIdentifyingExtension() *armcompute.VirtualMachineExtension { +func (p *DefaultProvider) getAKSIdentifyingExtension() *armcompute.VirtualMachineExtension { const ( vmExtensionType = "Microsoft.Compute/virtualMachines/extensions" aksIdentifyingExtensionName = "computeAksLinuxBilling" @@ -710,3 +721,11 @@ func ConvertToVirtualMachineIdentity(nodeIdentities []string) *armcompute.Virtua return identity } + +func getOfferingCapacityType(offering corecloudprovider.Offering) string { + return offering.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() +} + +func getOfferingZone(offering corecloudprovider.Offering) string { + return offering.Requirements.Get(v1.LabelTopologyZone).Any() +} diff --git a/pkg/providers/instance/instance_test.go b/pkg/providers/instance/instance_test.go index b777d5a48..5293da0fb 100644 --- a/pkg/providers/instance/instance_test.go +++ b/pkg/providers/instance/instance_test.go @@ -24,15 +24,17 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" "github.com/Azure/karpenter-provider-azure/pkg/cache" "github.com/stretchr/testify/assert" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + corev1 "k8s.io/api/core/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/scheduling" ) func TestGetPriorityCapacityAndInstanceType(t *testing.T) { cases := []struct { name string instanceTypes []*cloudprovider.InstanceType - nodeClaim *corev1beta1.NodeClaim + nodeClaim *karpv1.NodeClaim expectedInstanceType string expectedPriority string expectedZone string @@ -40,7 +42,7 @@ func TestGetPriorityCapacityAndInstanceType(t *testing.T) { { name: "No instance types in the list", instanceTypes: []*cloudprovider.InstanceType{}, - nodeClaim: &corev1beta1.NodeClaim{}, + nodeClaim: &karpv1.NodeClaim{}, expectedInstanceType: "", expectedPriority: "", expectedZone: "", @@ -52,10 +54,12 @@ func TestGetPriorityCapacityAndInstanceType(t *testing.T) { Name: "Standard_D2s_v3", Offerings: []cloudprovider.Offering{ { - Price: 0.1, - Zone: "westus-2", - CapacityType: corev1beta1.CapacityTypeOnDemand, - Available: true, + Price: 0.1, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-2"), + ), + Available: true, }, }, }, @@ -63,21 +67,23 @@ func TestGetPriorityCapacityAndInstanceType(t *testing.T) { Name: "Standard_NV16as_v4", Offerings: []cloudprovider.Offering{ { - Price: 0.1, - Zone: "westus-2", - CapacityType: corev1beta1.CapacityTypeOnDemand, - Available: true, + Price: 0.1, + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, "westus-2"), + ), + Available: true, }, }, }, }, - nodeClaim: &corev1beta1.NodeClaim{}, + nodeClaim: &karpv1.NodeClaim{}, expectedInstanceType: "Standard_D2s_v3", expectedZone: "2", - expectedPriority: corev1beta1.CapacityTypeOnDemand, + expectedPriority: karpv1.CapacityTypeOnDemand, }, } - provider := NewProvider(nil, nil, nil, nil, cache.NewUnavailableOfferings(), + provider := NewDefaultProvider(nil, nil, nil, nil, cache.NewUnavailableOfferings(), "westus-2", "MC_xxxxx_yyyy-region", "/subscriptions/0000000-0000-0000-0000-0000000000/resourceGroups/fake-resource-group-name/providers/Microsoft.Network/virtualNetworks/karpenter/subnets/nodesubnet", diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 9289a8377..8e2473ecc 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -21,6 +21,9 @@ import ( "strings" "testing" + "github.com/awslabs/operatorpkg/object" + opstatus "github.com/awslabs/operatorpkg/status" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" @@ -39,12 +42,12 @@ import ( "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" - "sigs.k8s.io/karpenter/pkg/operator/scheme" . "knative.dev/pkg/logging/testing" . "sigs.k8s.io/karpenter/pkg/test/expectations" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -67,7 +70,7 @@ func TestAzure(t *testing.T) { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx, stop = context.WithCancel(ctx) azureEnv = test.NewEnvironment(ctx, env) @@ -75,7 +78,7 @@ func TestAzure(t *testing.T) { cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) cloudProviderNonZonal = cloudprovider.New(azureEnvNonZonal.InstanceTypesProvider, azureEnvNonZonal.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnvNonZonal.ImageProvider) fakeClock = &clock.FakeClock{} - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) + cluster = state.NewCluster(fakeClock, env.Client) coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) RunSpecs(t, "Provider/Azure") } @@ -88,30 +91,33 @@ var _ = AfterSuite(func() { var _ = Describe("InstanceProvider", func() { var nodeClass *v1alpha2.AKSNodeClass - var nodePool *corev1beta1.NodePool - var nodeClaim *corev1beta1.NodeClaim + var nodePool *karpv1.NodePool + var nodeClaim *karpv1.NodeClaim BeforeEach(func() { nodeClass = test.AKSNodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, + nodeClass.StatusConditions().SetTrue(opstatus.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, }, }, }, }, }) - nodeClaim = coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim = coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: nodePool.Name, + karpv1.NodePoolLabelKey: nodePool.Name, }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -133,8 +139,8 @@ var _ = Describe("InstanceProvider", func() { func(azEnv *test.Environment, cp *cloudprovider.CloudProvider) { ExpectApplied(ctx, env.Client, nodeClaim, nodePool, nodeClass) for _, zone := range azEnv.Zones() { - azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeSpot) - azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeOnDemand) + azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeSpot) + azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeOnDemand) } instanceTypes, err := cp.GetInstanceTypes(ctx, nodePool) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index abc17cdea..fbbdfb902 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -23,13 +23,13 @@ import ( "github.com/Azure/skewer" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "knative.dev/pkg/ptr" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/utils" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" @@ -118,7 +118,7 @@ func (t TaxBrackets) Calculate(amount float64) float64 { return tax } -func NewInstanceType(ctx context.Context, sku *skewer.SKU, vmsize *skewer.VMSizeType, kc *corev1beta1.KubeletConfiguration, region string, +func NewInstanceType(ctx context.Context, sku *skewer.SKU, vmsize *skewer.VMSizeType, kc *v1alpha2.KubeletConfiguration, region string, offerings cloudprovider.Offerings, nodeClass *v1alpha2.AKSNodeClass, architecture string) *cloudprovider.InstanceType { return &cloudprovider.InstanceType{ Name: sku.GetName(), @@ -137,43 +137,41 @@ func computeRequirements(sku *skewer.SKU, vmsize *skewer.VMSizeType, architectur offerings cloudprovider.Offerings, region string) scheduling.Requirements { requirements := scheduling.NewRequirements( // Well Known Upstream - scheduling.NewRequirement(v1.LabelInstanceTypeStable, v1.NodeSelectorOpIn, sku.GetName()), - scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, getArchitecture(architecture)), - scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Linux)), - scheduling.NewRequirement( - v1.LabelTopologyZone, - v1.NodeSelectorOpIn, - lo.Map(offerings.Available(), - func(o cloudprovider.Offering, _ int) string { return o.Zone })...), - scheduling.NewRequirement(v1.LabelTopologyRegion, v1.NodeSelectorOpIn, region), + scheduling.NewRequirement(corev1.LabelInstanceTypeStable, corev1.NodeSelectorOpIn, sku.GetName()), + scheduling.NewRequirement(corev1.LabelArchStable, corev1.NodeSelectorOpIn, getArchitecture(architecture)), + scheduling.NewRequirement(corev1.LabelOSStable, corev1.NodeSelectorOpIn, string(corev1.Linux)), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { + return o.Requirements.Get(corev1.LabelTopologyZone).Any() + })...), + + scheduling.NewRequirement(corev1.LabelTopologyRegion, corev1.NodeSelectorOpIn, region), // Well Known to Karpenter - scheduling.NewRequirement( - corev1beta1.CapacityTypeLabelKey, - v1.NodeSelectorOpIn, - lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { + return o.Requirements.Get(karpv1.CapacityTypeLabelKey).Any() + })...), // Well Known to Azure - scheduling.NewRequirement(v1alpha2.LabelSKUCPU, v1.NodeSelectorOpIn, fmt.Sprint(vcpuCount(sku))), - scheduling.NewRequirement(v1alpha2.LabelSKUMemory, v1.NodeSelectorOpIn, fmt.Sprint((memoryMiB(sku)))), // in MiB - scheduling.NewRequirement(v1alpha2.LabelSKUGPUCount, v1.NodeSelectorOpIn, fmt.Sprint(gpuNvidiaCount(sku).Value())), - scheduling.NewRequirement(v1alpha2.LabelSKUGPUManufacturer, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUGPUName, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUCPU, corev1.NodeSelectorOpIn, fmt.Sprint(vcpuCount(sku))), + scheduling.NewRequirement(v1alpha2.LabelSKUMemory, corev1.NodeSelectorOpIn, fmt.Sprint((memoryMiB(sku)))), // in MiB + scheduling.NewRequirement(v1alpha2.LabelSKUGPUCount, corev1.NodeSelectorOpIn, fmt.Sprint(gpuNvidiaCount(sku).Value())), + scheduling.NewRequirement(v1alpha2.LabelSKUGPUManufacturer, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUGPUName, corev1.NodeSelectorOpDoesNotExist), // composites - scheduling.NewRequirement(v1alpha2.LabelSKUName, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUName, corev1.NodeSelectorOpDoesNotExist), // size parts - scheduling.NewRequirement(v1alpha2.LabelSKUFamily, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUAccelerator, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUVersion, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUFamily, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUAccelerator, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUVersion, corev1.NodeSelectorOpDoesNotExist), // SKU capabilities - scheduling.NewRequirement(v1alpha2.LabelSKUStorageEphemeralOSMaxSize, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUStoragePremiumCapable, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUEncryptionAtHostSupported, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUAcceleratedNetworking, v1.NodeSelectorOpDoesNotExist), - scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, v1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUStorageEphemeralOSMaxSize, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUStoragePremiumCapable, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUEncryptionAtHostSupported, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUAcceleratedNetworking, corev1.NodeSelectorOpDoesNotExist), + scheduling.NewRequirement(v1alpha2.LabelSKUHyperVGeneration, corev1.NodeSelectorOpDoesNotExist), // all additive feature initialized elsewhere ) @@ -263,13 +261,13 @@ func getArchitecture(architecture string) string { return architecture // unrecognized } -func computeCapacity(ctx context.Context, sku *skewer.SKU, kc *corev1beta1.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) v1.ResourceList { - return v1.ResourceList{ - v1.ResourceCPU: *cpu(sku), - v1.ResourceMemory: *memory(ctx, sku), - v1.ResourceEphemeralStorage: *ephemeralStorage(nodeClass), - v1.ResourcePods: *pods(sku, kc), - v1.ResourceName("nvidia.com/gpu"): *gpuNvidiaCount(sku), +func computeCapacity(ctx context.Context, sku *skewer.SKU, kc *v1alpha2.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) corev1.ResourceList { + return corev1.ResourceList{ + corev1.ResourceCPU: *cpu(sku), + corev1.ResourceMemory: *memory(ctx, sku), + corev1.ResourceEphemeralStorage: *ephemeralStorage(nodeClass), + corev1.ResourcePods: *pods(sku, kc), + corev1.ResourceName("nvidia.com/gpu"): *gpuNvidiaCount(sku), } } @@ -310,7 +308,7 @@ func ephemeralStorage(nodeClass *v1alpha2.AKSNodeClass) *resource.Quantity { return resource.NewScaledQuantity(int64(lo.FromPtr(nodeClass.Spec.OSDiskSizeGB)), resource.Giga) } -func pods(sku *skewer.SKU, kc *corev1beta1.KubeletConfiguration) *resource.Quantity { +func pods(sku *skewer.SKU, kc *v1alpha2.KubeletConfiguration) *resource.Quantity { // TODO: fine-tune pods calc var count int64 switch { @@ -325,29 +323,29 @@ func pods(sku *skewer.SKU, kc *corev1beta1.KubeletConfiguration) *resource.Quant return resources.Quantity(fmt.Sprint(count)) } -func SystemReservedResources() v1.ResourceList { +func SystemReservedResources() corev1.ResourceList { // AKS does not set system-reserved values and only CPU and memory are considered // https://learn.microsoft.com/en-us/azure/aks/concepts-clusters-workloads#resource-reservations - return v1.ResourceList{ - v1.ResourceCPU: resource.Quantity{}, - v1.ResourceMemory: resource.Quantity{}, + return corev1.ResourceList{ + corev1.ResourceCPU: resource.Quantity{}, + corev1.ResourceMemory: resource.Quantity{}, } } -func KubeReservedResources(vcpus int64, memoryGib float64) v1.ResourceList { +func KubeReservedResources(vcpus int64, memoryGib float64) corev1.ResourceList { reservedMemoryMi := int64(1024 * reservedMemoryTaxGi.Calculate(memoryGib)) reservedCPUMilli := int64(1000 * reservedCPUTaxVCPU.Calculate(float64(vcpus))) - resources := v1.ResourceList{ - v1.ResourceCPU: *resource.NewScaledQuantity(reservedCPUMilli, resource.Milli), - v1.ResourceMemory: *resource.NewQuantity(reservedMemoryMi*1024*1024, resource.BinarySI), + resources := corev1.ResourceList{ + corev1.ResourceCPU: *resource.NewScaledQuantity(reservedCPUMilli, resource.Milli), + corev1.ResourceMemory: *resource.NewQuantity(reservedMemoryMi*1024*1024, resource.BinarySI), } return resources } -func EvictionThreshold() v1.ResourceList { - return v1.ResourceList{ - v1.ResourceMemory: resource.MustParse(DefaultMemoryAvailable), +func EvictionThreshold() corev1.ResourceList { + return corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse(DefaultMemoryAvailable), } } diff --git a/pkg/providers/instancetype/instancetypes.go b/pkg/providers/instancetype/instancetypes.go index 8e2addb5b..a463bdc3b 100644 --- a/pkg/providers/instancetype/instancetypes.go +++ b/pkg/providers/instancetype/instancetypes.go @@ -29,6 +29,9 @@ import ( "github.com/mitchellh/hashstructure/v2" "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "github.com/Azure/go-autorest/autorest/to" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" kcache "github.com/Azure/karpenter-provider-azure/pkg/cache" @@ -42,8 +45,8 @@ import ( "github.com/Azure/skewer" "github.com/alecthomas/units" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/utils/pretty" ) @@ -52,7 +55,17 @@ const ( InstanceTypesCacheTTL = 23 * time.Hour ) -type Provider struct { +type Provider interface { + LivenessProbe(*http.Request) error + List(context.Context, *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) + //UpdateInstanceTypes(ctx context.Context) error + //UpdateInstanceTypeOfferings(ctx context.Context) error +} + +// assert that DefaultProvider implements Provider interface +var _ Provider = (*DefaultProvider)(nil) + +type DefaultProvider struct { region string skuClient skuclient.SkuClient pricingProvider *pricing.Provider @@ -62,30 +75,32 @@ type Provider struct { // Values cached *before* considering insufficient capacity errors from the unavailableOfferings cache. // Fully initialized Instance Types are also cached based on the set of all instance types, // unavailableOfferings cache, AWSNodeClass, and kubelet configuration from the NodePool - mu sync.Mutex - cache *cache.Cache + mu sync.Mutex + instanceTypesCache *cache.Cache cm *pretty.ChangeMonitor // instanceTypesSeqNum is a monotonically increasing change counter used to avoid the expensive hashing operation on instance types instanceTypesSeqNum uint64 } -func NewProvider(region string, cache *cache.Cache, skuClient skuclient.SkuClient, pricingProvider *pricing.Provider, offeringsCache *kcache.UnavailableOfferings) *Provider { - return &Provider{ +func NewDefaultProvider(region string, cache *cache.Cache, skuClient skuclient.SkuClient, pricingProvider *pricing.Provider, offeringsCache *kcache.UnavailableOfferings) *DefaultProvider { + return &DefaultProvider{ // TODO: skewer api, subnetprovider, pricing provider, unavailable offerings, ... region: region, skuClient: skuClient, pricingProvider: pricingProvider, unavailableOfferings: offeringsCache, - cache: cache, + instanceTypesCache: cache, cm: pretty.NewChangeMonitor(), instanceTypesSeqNum: 0, } } // Get all instance type options -func (p *Provider) List( - ctx context.Context, kc *corev1beta1.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { +func (p *DefaultProvider) List( + ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) ([]*cloudprovider.InstanceType, error) { + kc := nodeClass.Spec.Kubelet + // Get SKUs from Azure skus, err := p.getInstanceTypes(ctx) if err != nil { @@ -101,8 +116,10 @@ func (p *Provider) List( to.String(nodeClass.Spec.ImageFamily), to.Int32(nodeClass.Spec.OSDiskSizeGB), ) - if item, ok := p.cache.Get(key); ok { - return item.([]*cloudprovider.InstanceType), nil + if item, ok := p.instanceTypesCache.Get(key); ok { + // Ensure what's returned from this function is a shallow-copy of the slice (not a deep-copy of the data itself) + // so that modifications to the ordering of the data don't affect the original + return append([]*cloudprovider.InstanceType{}, item.([]*cloudprovider.InstanceType)...), nil } // Get Viable offerings @@ -120,6 +137,10 @@ func (p *Provider) List( continue } instanceTypeZones := instanceTypeZones(sku, p.region) + // !!! Important !!! + // Any changes to the values passed into the NewInstanceType method will require making updates to the cache key + // so that Karpenter is able to cache the set of InstanceTypes based on values that alter the set of instance types + // !!! Important !!! instanceType := NewInstanceType(ctx, sku, vmsize, kc, p.region, p.createOfferings(sku, instanceTypeZones), nodeClass, architecture) if len(instanceType.Offerings) == 0 { continue @@ -131,11 +152,11 @@ func (p *Provider) List( result = append(result, instanceType) } - p.cache.SetDefault(key, result) + p.instanceTypesCache.SetDefault(key, result) return result, nil } -func (p *Provider) LivenessProbe(req *http.Request) error { +func (p *DefaultProvider) LivenessProbe(req *http.Request) error { return p.pricingProvider.LivenessProbe(req) } @@ -155,20 +176,61 @@ func instanceTypeZones(sku *skewer.SKU, region string) sets.Set[string] { return sets.New("") // empty string means non-zonal offering } -func (p *Provider) createOfferings(sku *skewer.SKU, zones sets.Set[string]) []cloudprovider.Offering { +// TODO: review; switch to controller-driven updates +// createOfferings creates a set of mutually exclusive offerings for a given instance type. This provider maintains an +// invariant that each offering is mutually exclusive. Specifically, there is an offering for each permutation of zone +// and capacity type. ZoneID is also injected into the offering requirements, when available, but there is a 1-1 +// mapping between zone and zoneID so this does not change the number of offerings. +// +// Each requirement on the offering is guaranteed to have a single value. To get the value for a requirement on an +// offering, you can do the following thanks to this invariant: +// +// offering.Requirements.Get(v1.TopologyLabelZone).Any() +func (p *DefaultProvider) createOfferings(sku *skewer.SKU, zones sets.Set[string]) []cloudprovider.Offering { offerings := []cloudprovider.Offering{} for zone := range zones { onDemandPrice, onDemandOk := p.pricingProvider.OnDemandPrice(*sku.Name) spotPrice, spotOk := p.pricingProvider.SpotPrice(*sku.Name) - availableOnDemand := onDemandOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, corev1beta1.CapacityTypeOnDemand) - availableSpot := spotOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, corev1beta1.CapacityTypeSpot) - offerings = append(offerings, cloudprovider.Offering{Zone: zone, CapacityType: corev1beta1.CapacityTypeSpot, Price: spotPrice, Available: availableSpot}) - offerings = append(offerings, cloudprovider.Offering{Zone: zone, CapacityType: corev1beta1.CapacityTypeOnDemand, Price: onDemandPrice, Available: availableOnDemand}) + availableOnDemand := onDemandOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, karpv1.CapacityTypeOnDemand) + availableSpot := spotOk && !p.unavailableOfferings.IsUnavailable(*sku.Name, zone, karpv1.CapacityTypeSpot) + + onDemandOffering := cloudprovider.Offering{ + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeOnDemand), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zone), + ), + Price: onDemandPrice, + Available: availableOnDemand, + } + + spotOffering := cloudprovider.Offering{ + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(karpv1.CapacityTypeLabelKey, corev1.NodeSelectorOpIn, karpv1.CapacityTypeSpot), + scheduling.NewRequirement(corev1.LabelTopologyZone, corev1.NodeSelectorOpIn, zone), + ), + Price: spotPrice, + Available: availableSpot, + } + + offerings = append(offerings, onDemandOffering, spotOffering) + + /* + instanceTypeOfferingAvailable.With(prometheus.Labels{ + instanceTypeLabel: *instanceType.InstanceType, + capacityTypeLabel: capacityType, + zoneLabel: zone, + }).Set(float64(lo.Ternary(available, 1, 0))) + instanceTypeOfferingPriceEstimate.With(prometheus.Labels{ + instanceTypeLabel: *instanceType.InstanceType, + capacityTypeLabel: capacityType, + zoneLabel: zone, + }).Set(price) + */ } return offerings } -func (p *Provider) isInstanceTypeSupportedByImageFamily(skuName, imageFamily string) bool { +func (p *DefaultProvider) isInstanceTypeSupportedByImageFamily(skuName, imageFamily string) bool { // Currently only GPU has conditional support by image family if !(utils.IsNvidiaEnabledSKU(skuName) || utils.IsMarinerEnabledGPUSKU(skuName)) { return true @@ -184,7 +246,7 @@ func (p *Provider) isInstanceTypeSupportedByImageFamily(skuName, imageFamily str } // getInstanceTypes retrieves all instance types from skewer using some opinionated filters -func (p *Provider) getInstanceTypes(ctx context.Context) (map[string]*skewer.SKU, error) { +func (p *DefaultProvider) getInstanceTypes(ctx context.Context) (map[string]*skewer.SKU, error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to GetInstanceTypes do not result in cache misses and multiple // calls to Resource API when we could have just made one call. This lock is here because multiple callers result @@ -193,7 +255,7 @@ func (p *Provider) getInstanceTypes(ctx context.Context) (map[string]*skewer.SKU p.mu.Lock() defer p.mu.Unlock() - if cached, ok := p.cache.Get(InstanceTypesCacheKey); ok { + if cached, ok := p.instanceTypesCache.Get(InstanceTypesCacheKey); ok { return cached.(map[string]*skewer.SKU), nil } instanceTypes := map[string]*skewer.SKU{} @@ -224,12 +286,12 @@ func (p *Provider) getInstanceTypes(ctx context.Context) (map[string]*skewer.SKU logging.FromContext(ctx).With( "count", len(instanceTypes)).Debugf("discovered instance types") } - p.cache.SetDefault(InstanceTypesCacheKey, instanceTypes) + p.instanceTypesCache.SetDefault(InstanceTypesCacheKey, instanceTypes) return instanceTypes, nil } // isSupported indicates SKU is supported by AKS, based on SKU properties -func (p *Provider) isSupported(sku *skewer.SKU, vmsize *skewer.VMSizeType) bool { +func (p *DefaultProvider) isSupported(sku *skewer.SKU, vmsize *skewer.VMSizeType) bool { return p.hasMinimumCPU(sku) && p.hasMinimumMemory(sku) && !p.isUnsupportedByAKS(sku) && @@ -239,24 +301,24 @@ func (p *Provider) isSupported(sku *skewer.SKU, vmsize *skewer.VMSizeType) bool } // at least 2 cpus -func (p *Provider) hasMinimumCPU(sku *skewer.SKU) bool { +func (p *DefaultProvider) hasMinimumCPU(sku *skewer.SKU) bool { cpu, err := sku.VCPU() return err == nil && cpu >= 2 } // at least 3.5 GiB of memory -func (p *Provider) hasMinimumMemory(sku *skewer.SKU) bool { +func (p *DefaultProvider) hasMinimumMemory(sku *skewer.SKU) bool { memGiB, err := sku.Memory() return err == nil && memGiB >= 3.5 } // instances AKS does not support -func (p *Provider) isUnsupportedByAKS(sku *skewer.SKU) bool { +func (p *DefaultProvider) isUnsupportedByAKS(sku *skewer.SKU) bool { return RestrictedVMSizes.Has(sku.GetName()) } // GPU SKUs AKS does not support -func (p *Provider) isUnsupportedGPU(sku *skewer.SKU) bool { +func (p *DefaultProvider) isUnsupportedGPU(sku *skewer.SKU) bool { name := lo.FromPtr(sku.Name) gpu, err := sku.GPU() if err != nil || gpu <= 0 { @@ -266,12 +328,12 @@ func (p *Provider) isUnsupportedGPU(sku *skewer.SKU) bool { } // SKU with constrained CPUs -func (p *Provider) hasConstrainedCPUs(vmsize *skewer.VMSizeType) bool { +func (p *DefaultProvider) hasConstrainedCPUs(vmsize *skewer.VMSizeType) bool { return vmsize.CpusConstrained != nil } // confidential VMs (DC, EC) are not yet supported by this Karpenter provider -func (p *Provider) isConfidential(sku *skewer.SKU) bool { +func (p *DefaultProvider) isConfidential(sku *skewer.SKU) bool { size := sku.GetSize() return strings.HasPrefix(size, "DC") || strings.HasPrefix(size, "EC") } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 05570b78e..331a0c8a4 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -27,6 +27,7 @@ import ( "testing" "time" + "github.com/awslabs/operatorpkg/status" "github.com/blang/semver/v4" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -41,14 +42,14 @@ import ( coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" "sigs.k8s.io/karpenter/pkg/controllers/state" "sigs.k8s.io/karpenter/pkg/events" - "sigs.k8s.io/karpenter/pkg/operator/scheme" coretest "sigs.k8s.io/karpenter/pkg/test" . "sigs.k8s.io/karpenter/pkg/test/expectations" + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" sdkerrors "github.com/Azure/azure-sdk-for-go-extensions/pkg/errors" "github.com/Azure/azure-sdk-for-go/sdk/azcore" @@ -84,7 +85,7 @@ func TestAzure(t *testing.T) { ctx = coreoptions.ToContext(ctx, coretest.Options()) ctx = options.ToContext(ctx, test.Options()) - env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...)) ctx, stop = context.WithCancel(ctx) azureEnv = test.NewEnvironment(ctx, env) @@ -94,8 +95,8 @@ func TestAzure(t *testing.T) { cloudProvider = cloudprovider.New(azureEnv.InstanceTypesProvider, azureEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnv.ImageProvider) cloudProviderNonZonal = cloudprovider.New(azureEnvNonZonal.InstanceTypesProvider, azureEnvNonZonal.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, azureEnvNonZonal.ImageProvider) - cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - clusterNonZonal = state.NewCluster(fakeClock, env.Client, cloudProviderNonZonal) + cluster = state.NewCluster(fakeClock, env.Client) + clusterNonZonal = state.NewCluster(fakeClock, env.Client) coreProvisioner = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) coreProvisionerNonZonal = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProviderNonZonal, clusterNonZonal) @@ -112,15 +113,16 @@ var _ = AfterSuite(func() { var _ = Describe("InstanceType Provider", func() { var nodeClass *v1alpha2.AKSNodeClass - var nodePool *corev1beta1.NodePool + var nodePool *karpv1.NodePool BeforeEach(func() { nodeClass = test.AKSNodeClass() - nodePool = coretest.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ + nodeClass.StatusConditions().SetTrue(status.ConditionReady) + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -202,11 +204,11 @@ var _ = Describe("InstanceType Provider", func() { It("should fail to provision when LowPriorityCoresQuota errors are hit, then switch capacity type and succeed", func() { LowPriorityCoresQuotaErrorMessage := "Operation could not be completed as it results in exceeding approved Low Priority Cores quota. Additional details - Deployment Model: Resource Manager, Location: westus2, Current Limit: 0, Current Usage: 0, Additional Required: 32, (Minimum) New Limit Required: 32. Submit a request for Quota increase at https://aka.ms/ProdportalCRP/#blade/Microsoft_Azure_Capacity/UsageAndQuota.ReactView/Parameters/%7B%22subscriptionId%22:%(redacted)%22,%22command%22:%22openQuotaApprovalBlade%22,%22quotas%22:[%7B%22location%22:%22westus2%22,%22providerId%22:%22Microsoft.Compute%22,%22resourceName%22:%22LowPriorityCores%22,%22quotaRequest%22:%7B%22properties%22:%7B%22limit%22:32,%22unit%22:%22Count%22,%22name%22:%7B%22value%22:%22LowPriorityCores%22%7D%7D%7D%7D]%7D by specifying parameters listed in the ‘Details’ section for deployment to succeed. Please read more about quota limits at https://docs.microsoft.com/en-us/azure/azure-supportability/per-vm-quota-requests" // Create nodepool that has both ondemand and spot capacity types enabled - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, + Key: karpv1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand, corev1beta1.CapacityTypeSpot}, + Values: []string{karpv1.CapacityTypeOnDemand, karpv1.CapacityTypeSpot}, }}) ExpectApplied(ctx, env.Client, nodePool, nodeClass) // Set the LowPriorityCoresQuota error to be returned when creating the vm @@ -230,7 +232,7 @@ var _ = Describe("InstanceType Provider", func() { nodes, err := env.KubernetesInterface.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(len(nodes.Items)).To(Equal(1)) - Expect(nodes.Items[0].Labels[corev1beta1.CapacityTypeLabelKey]).To(Equal(corev1beta1.CapacityTypeOnDemand)) + Expect(nodes.Items[0].Labels[karpv1.CapacityTypeLabelKey]).To(Equal(karpv1.CapacityTypeOnDemand)) }) It("should fail to provision when VM SKU family vCPU quota exceeded error is returned, and succeed when it is gone", func() { @@ -301,14 +303,14 @@ var _ = Describe("InstanceType Provider", func() { ) ExpectApplied(ctx, env.Client, nodePool, nodeClass) - nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ - corev1beta1.NodePoolLabelKey: nodePool.Name, + karpv1.NodePoolLabelKey: nodePool.Name, }, }, - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -326,7 +328,7 @@ var _ = Describe("InstanceType Provider", func() { getName := func(instanceType *corecloudprovider.InstanceType) string { return instanceType.Name } BeforeEach(func() { - instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) }) @@ -354,7 +356,7 @@ var _ = Describe("InstanceType Provider", func() { nodeClassAZLinux := test.AKSNodeClass() nodeClassAZLinux.Spec.ImageFamily = lo.ToPtr("AzureLinux") ExpectApplied(ctx, env.Client, nodeClassAZLinux) - instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClassAZLinux) + instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, nodeClassAZLinux) Expect(err).ToNot(HaveOccurred()) }) @@ -371,13 +373,13 @@ var _ = Describe("InstanceType Provider", func() { // Create a Provisioner that selects a sku that supports ephemeral // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 128GB np := coretest.NodePool() - np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: "node.kubernetes.io/instance-type", Operator: v1.NodeSelectorOpIn, Values: []string{"Standard_D64s_v3"}, }}) - np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ Name: nodeClass.Name, } @@ -398,20 +400,21 @@ var _ = Describe("InstanceType Provider", func() { It("should use ephemeral disk if supported, and set disk size to OSDiskSizeGB from node class", func() { // Create a Nodepool that selects a sku that supports ephemeral // SKU Standard_D64s_v3 has 1600GB of CacheDisk space, so we expect we can create an ephemeral disk with size 256GB - provider := test.AKSNodeClass() - provider.Spec.OSDiskSizeGB = lo.ToPtr[int32](256) + nodeClass = test.AKSNodeClass() + nodeClass.Spec.OSDiskSizeGB = lo.ToPtr[int32](256) + nodeClass.StatusConditions().SetTrue(status.ConditionReady) np := coretest.NodePool() - np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: "node.kubernetes.io/instance-type", Operator: v1.NodeSelectorOpIn, Values: []string{"Standard_D64s_v3"}, }}) - np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ - Name: provider.Name, + np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Name: nodeClass.Name, } - ExpectApplied(ctx, env.Client, np, provider) + ExpectApplied(ctx, env.Client, np, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) @@ -428,13 +431,13 @@ var _ = Describe("InstanceType Provider", func() { // and has 16GB of Temp Disk Space. // With our rule of 100GB being the minimum OSDiskSize, this VM should be created without local disk np := coretest.NodePool() - np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + np.Spec.Template.Spec.Requirements = append(np.Spec.Template.Spec.Requirements, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: "node.kubernetes.io/instance-type", Operator: v1.NodeSelectorOpIn, Values: []string{"Standard_D2s_v3"}, }}) - np.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ + np.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ Name: nodeClass.Name, } @@ -452,7 +455,7 @@ var _ = Describe("InstanceType Provider", func() { Context("Nodepool with KubeletConfig", func() { It("should support provisioning with kubeletConfig, computeResources and maxPods not specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ PodsPerCore: lo.ToPtr(int32(110)), EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "1Gi", @@ -470,15 +473,7 @@ var _ = Describe("InstanceType Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) - - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString := string(decodedBytes[:]) - kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + kubeletFlags := ExpectKubeletFlagsPassed() Expect(kubeletFlags).To(SatisfyAny( // AKS default ContainSubstring("--system-reserved=cpu=0,memory=0"), @@ -516,7 +511,7 @@ var _ = Describe("InstanceType Provider", func() { }) It("should support provisioning with kubeletConfig, computeResources and maxPods not specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ PodsPerCore: lo.ToPtr(int32(110)), EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "1Gi", @@ -534,15 +529,7 @@ var _ = Describe("InstanceType Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) - - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString := string(decodedBytes[:]) - kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + kubeletFlags := ExpectKubeletFlagsPassed() Expect(kubeletFlags).To(SatisfyAny( // AKS default ContainSubstring("--system-reserved=cpu=0,memory=0"), @@ -562,7 +549,7 @@ var _ = Describe("InstanceType Provider", func() { Expect(kubeletFlags).To(ContainSubstring("--cpu-cfs-quota=true")) }) It("should support provisioning with kubeletConfig, computeResources and maxPods specified", func() { - nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ PodsPerCore: lo.ToPtr(int32(110)), EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "1Gi", @@ -594,14 +581,7 @@ var _ = Describe("InstanceType Provider", func() { ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString := string(decodedBytes[:]) - kubeletFlags := decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + kubeletFlags := ExpectKubeletFlagsPassed() Expect(kubeletFlags).To(SatisfyAny( // AKS default ContainSubstring("--system-reserved=cpu=0,memory=0"), @@ -624,9 +604,9 @@ var _ = Describe("InstanceType Provider", func() { Context("Unavailable Offerings", func() { It("should not allocate a vm in a zone marked as unavailable", func() { - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeSpot) - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeOnDemand) - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeOnDemand) + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, @@ -653,10 +633,10 @@ var _ = Describe("InstanceType Provider", func() { DescribeTable("Should not return unavailable offerings", func(azEnv *test.Environment) { for _, zone := range azEnv.Zones() { - azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeSpot) - azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeOnDemand) + azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeSpot) + azEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeOnDemand) } - instanceTypes, err := azEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + instanceTypes, err := azEnv.InstanceTypesProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) seeUnavailable := false @@ -678,8 +658,8 @@ var _ = Describe("InstanceType Provider", func() { ) It("should launch instances in a different zone than preferred", func() { - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeOnDemand) - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeOnDemand) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "ZonalAllocationFailure", "Standard_D2_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeSpot) ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -698,9 +678,9 @@ var _ = Describe("InstanceType Provider", func() { Expect(node.Labels["node.kubernetes.io/instance-type"]).To(Equal("Standard_D2_v2")) }) It("should launch smaller instances than optimal if larger instance launch results in Insufficient Capacity Error", func() { - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeOnDemand) - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), corev1beta1.CapacityTypeSpot) - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeOnDemand) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_F16s_v2", fmt.Sprintf("%s-1", fake.Region), karpv1.CapacityTypeSpot) + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, @@ -731,8 +711,8 @@ var _ = Describe("InstanceType Provider", func() { DescribeTable("should launch instances on later reconciliation attempt with Insufficient Capacity Error Cache expiry", func(azureEnv *test.Environment, cluster *state.Cluster, cloudProvider *cloudprovider.CloudProvider, coreProvisioner *provisioning.Provisioner) { for _, zone := range azureEnv.Zones() { - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeSpot) - azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, corev1beta1.CapacityTypeOnDemand) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeSpot) + azureEnv.UnavailableOfferingsCache.MarkUnavailable(ctx, "SubscriptionQuotaReached", "Standard_D2_v2", zone, karpv1.CapacityTypeOnDemand) } ExpectApplied(ctx, env.Client, nodeClass, nodePool) @@ -758,10 +738,10 @@ var _ = Describe("InstanceType Provider", func() { &azcore.ResponseError{ErrorCode: sdkerrors.SKUNotAvailableErrorCode}, ) coretest.ReplaceRequirements(nodePool, - corev1beta1.NodeSelectorRequirementWithMinValues{ + karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{sku}}}, - corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{capacityType}}}, + karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{capacityType}}}, ) ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() @@ -773,11 +753,11 @@ var _ = Describe("InstanceType Provider", func() { } It("should mark SKU as unavailable in all zones for Spot", func() { - AssertUnavailable("Standard_D2_v2", corev1beta1.CapacityTypeSpot) + AssertUnavailable("Standard_D2_v2", karpv1.CapacityTypeSpot) }) It("should mark SKU as unavailable in all zones for OnDemand", func() { - AssertUnavailable("Standard_D2_v2", corev1beta1.CapacityTypeOnDemand) + AssertUnavailable("Standard_D2_v2", karpv1.CapacityTypeOnDemand) }) }) }) @@ -790,7 +770,7 @@ var _ = Describe("InstanceType Provider", func() { ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) - instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, &corev1beta1.KubeletConfiguration{}, nodeClass) + instanceTypes, err = azureEnv.InstanceTypesProvider.List(ctx, nodeClass) Expect(err).ToNot(HaveOccurred()) }) @@ -827,13 +807,13 @@ var _ = Describe("InstanceType Provider", func() { nodeSelector := map[string]string{ // Well known - v1.LabelTopologyRegion: fake.Region, - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyZone: fmt.Sprintf("%s-1", fake.Region), - v1.LabelInstanceTypeStable: "Standard_NC24ads_A100_v4", - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: "on-demand", + v1.LabelTopologyRegion: fake.Region, + karpv1.NodePoolLabelKey: nodePool.Name, + v1.LabelTopologyZone: fmt.Sprintf("%s-1", fake.Region), + v1.LabelInstanceTypeStable: "Standard_NC24ads_A100_v4", + v1.LabelOSStable: "linux", + v1.LabelArchStable: "amd64", + karpv1.CapacityTypeLabelKey: "on-demand", // Well Known to AKS v1alpha2.LabelSKUName: "Standard_NC24ads_A100_v4", v1alpha2.LabelSKUFamily: "N", @@ -861,7 +841,7 @@ var _ = Describe("InstanceType Provider", func() { } // Ensure that we're exercising all well known labels - Expect(lo.Keys(nodeSelector)).To(ContainElements(append(corev1beta1.WellKnownLabels.UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...))) + Expect(lo.Keys(nodeSelector)).To(ContainElements(append(karpv1.WellKnownLabels.UnsortedList(), lo.Keys(karpv1.NormalizedLabels)...))) var pods []*v1.Pod for key, value := range nodeSelector { @@ -926,13 +906,13 @@ var _ = Describe("InstanceType Provider", func() { DescribeTable("should select the right image for a given instance type", func(instanceType string, imageFamily string, expectedImageDefinition string, expectedGalleryURL string) { nodeClass.Spec.ImageFamily = lo.ToPtr(imageFamily) - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{instanceType}, }}) - nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{Name: nodeClass.Name} + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{Name: nodeClass.Name} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{}) ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) @@ -1104,15 +1084,7 @@ var _ = Describe("InstanceType Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) - - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString = string(decodedBytes[:]) - kubeletFlags = decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS="):] + kubeletFlags = ExpectKubeletFlagsPassed() k8sVersion, err := azureEnv.ImageProvider.KubeServerVersion(ctx) Expect(err).To(BeNil()) @@ -1144,6 +1116,10 @@ var _ = Describe("InstanceType Provider", func() { Expect(kubeletFlags).ToNot(ContainSubstring("--image-credential-provider-bin-dir")) } }) + + It("should include karpenter.sh/unregistered taint", func() { + Expect(kubeletFlags).To(ContainSubstring("--register-with-taints=" + karpv1.UnregisteredNoExecuteTaint.ToString())) + }) }) Context("LoadBalancer", func() { resourceGroup := "test-resourceGroup" @@ -1177,8 +1153,8 @@ var _ = Describe("InstanceType Provider", func() { Context("Zone-aware provisioning", func() { It("should launch in the NodePool-requested zone", func() { zone, vmZone := "eastus-3", "3" - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}, + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: karpv1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{karpv1.CapacityTypeSpot, karpv1.CapacityTypeOnDemand}}}, {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{zone}}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1202,7 +1178,7 @@ var _ = Describe("InstanceType Provider", func() { Expect(vm.Zones).To(BeEmpty()) }) It("should support provisioning non-zonal instance types in zonal regions", func() { - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, @@ -1272,3 +1248,15 @@ var _ = Describe("Tax Calculator", func() { func createSDKErrorBody(code, message string) io.ReadCloser { return io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"error":{"code": "%s", "message": "%s"}}`, code, message)))) } + +func ExpectKubeletFlagsPassed() string { + GinkgoHelper() + Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) + vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM + customData := *vm.Properties.OSProfile.CustomData + Expect(customData).ToNot(BeNil()) + decodedBytes, err := base64.StdEncoding.DecodeString(customData) + Expect(err).To(Succeed()) + decodedString := string(decodedBytes[:]) + return decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS=") : strings.Index(decodedString, "KUBELET_NODE_LABELS")] +} diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 22c52e30f..ed8783df8 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -29,7 +29,7 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/operator/options" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/scheduling" ) @@ -84,7 +84,7 @@ func NewProvider(_ context.Context, imageFamily *imagefamily.Resolver, imageProv } } -func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *corev1beta1.NodeClaim, +func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, nodeClaim *karpv1.NodeClaim, instanceType *cloudprovider.InstanceType, additionalLabels map[string]string) (*Template, error) { staticParameters, err := p.getStaticParameters(ctx, instanceType, nodeClass, lo.Assign(nodeClaim.Labels, additionalLabels)) if err != nil { @@ -109,9 +109,9 @@ func (p *Provider) GetTemplate(ctx context.Context, nodeClass *v1alpha2.AKSNodeC } func (p *Provider) getStaticParameters(ctx context.Context, instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass, labels map[string]string) (*parameters.StaticParameters, error) { - var arch string = corev1beta1.ArchitectureAmd64 - if err := instanceType.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64))); err == nil { - arch = corev1beta1.ArchitectureArm64 + var arch string = karpv1.ArchitectureAmd64 + if err := instanceType.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, karpv1.ArchitectureArm64))); err == nil { + arch = karpv1.ArchitectureArm64 } // TODO: make conditional on either Azure CNI Overlay or pod subnet vnetLabels, err := p.getVnetInfoLabels(ctx, nodeClass) From 95f4055a7ec4eeb37ce438e09807be7decadaa24 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:00:10 +0000 Subject: [PATCH 17/47] chore: migrate test pkg to v1 API --- pkg/test/environment.go | 21 ++++++++++----------- pkg/test/nodepool.go | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 4d393da02..4b2ea5fa7 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -20,8 +20,11 @@ import ( "context" "github.com/samber/lo" + corev1 "k8s.io/api/core/v1" + + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "github.com/Azure/karpenter-provider-azure/pkg/apis" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" "github.com/Azure/karpenter-provider-azure/pkg/fake" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" @@ -31,18 +34,14 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/providers/loadbalancer" "github.com/Azure/karpenter-provider-azure/pkg/providers/pricing" "github.com/patrickmn/go-cache" - corev1 "k8s.io/api/core/v1" "knative.dev/pkg/ptr" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/operator/scheme" - coretest "sigs.k8s.io/karpenter/pkg/test" ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) + karpv1beta1.NormalizedLabels = lo.Assign(karpv1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": corev1.LabelTopologyZone}) } var ( @@ -67,8 +66,8 @@ type Environment struct { UnavailableOfferingsCache *azurecache.UnavailableOfferings // Providers - InstanceTypesProvider *instancetype.Provider - InstanceProvider *instance.Provider + InstanceTypesProvider instancetype.Provider + InstanceProvider instance.Provider PricingProvider *pricing.Provider ImageProvider *imagefamily.Provider ImageResolver *imagefamily.Resolver @@ -110,7 +109,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi pricingProvider := pricing.NewProvider(ctx, pricingAPI, region, make(chan struct{})) imageFamilyProvider := imagefamily.NewProvider(env.KubernetesInterface, kubernetesVersionCache, communityImageVersionsAPI, region) imageFamilyResolver := imagefamily.New(env.Client, imageFamilyProvider) - instanceTypesProvider := instancetype.NewProvider(region, instanceTypeCache, skuClientSingleton, pricingProvider, unavailableOfferingsCache) + instanceTypesProvider := instancetype.NewDefaultProvider(region, instanceTypeCache, skuClientSingleton, pricingProvider, unavailableOfferingsCache) launchTemplateProvider := launchtemplate.NewProvider( ctx, imageFamilyResolver, @@ -138,7 +137,7 @@ func NewRegionalEnvironment(ctx context.Context, env *coretest.Environment, regi communityImageVersionsAPI, skuClientSingleton, ) - instanceProvider := instance.NewProvider( + instanceProvider := instance.NewDefaultProvider( azClient, instanceTypesProvider, launchTemplateProvider, diff --git a/pkg/test/nodepool.go b/pkg/test/nodepool.go index fc7587bec..056420442 100644 --- a/pkg/test/nodepool.go +++ b/pkg/test/nodepool.go @@ -19,11 +19,11 @@ package test import ( "context" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) -func NodePool(options corev1beta1.NodePool) *corev1beta1.NodePool { +func NodePool(options karpv1.NodePool) *karpv1.NodePool { nodePool := test.NodePool(options) nodePool.SetDefaults(context.Background()) return nodePool From 03bbfc6ce3ad4d9b086ed69ec2ef770e3fec6a31 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:00:25 +0000 Subject: [PATCH 18/47] chore: update utils --- pkg/utils/utils.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index bd6ea31de..58eecc595 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -19,8 +19,12 @@ package utils import ( "context" "fmt" + "os" "regexp" + "strconv" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" + v1 "k8s.io/api/core/v1" "knative.dev/pkg/logging" "sigs.k8s.io/cloud-provider-azure/pkg/provider" ) @@ -58,3 +62,62 @@ func MkVMID(resourceGroupName string, vmName string) string { const idFormat = "/subscriptions/subscriptionID/resourceGroups/%s/providers/Microsoft.Compute/virtualMachines/%s" return fmt.Sprintf(idFormat, resourceGroupName, vmName) } + +// WithDefaultFloat64 returns the float64 value of the supplied environment variable or, if not present, +// the supplied default value. If the float64 conversion fails, returns the default +func WithDefaultFloat64(key string, def float64) float64 { + val, ok := os.LookupEnv(key) + if !ok { + return def + } + f, err := strconv.ParseFloat(val, 64) + if err != nil { + return def + } + return f +} + +func ImageReferenceToString(imageRef armcompute.ImageReference) string { + // Check for Custom Image + if imageRef.ID != nil && *imageRef.ID != "" { + return *imageRef.ID + } + + // Check for Community Image + if imageRef.CommunityGalleryImageID != nil && *imageRef.CommunityGalleryImageID != "" { + return *imageRef.CommunityGalleryImageID + } + + // Check for Shared Gallery Image + if imageRef.SharedGalleryImageID != nil && *imageRef.SharedGalleryImageID != "" { + return *imageRef.SharedGalleryImageID + } + + // Check for Platform Image and use standard string representation + if imageRef.Publisher != nil && imageRef.Offer != nil && imageRef.SKU != nil && imageRef.Version != nil { + // Use the standard format: Publisher:Offer:Sku:Version + return fmt.Sprintf("%s:%s:%s:%s", + *imageRef.Publisher, *imageRef.Offer, *imageRef.SKU, *imageRef.Version) + } + + return "" +} + +func IsVMDeleting(vm armcompute.VirtualMachine) bool { + if vm.Properties != nil && vm.Properties.ProvisioningState != nil { + return *vm.Properties.ProvisioningState == "Deleting" + } + return false +} + +// StringMap returns the string map representation of the resource list +func StringMap(list v1.ResourceList) map[string]string { + if list == nil { + return nil + } + m := make(map[string]string) + for k, v := range list { + m[k.String()] = v.String() + } + return m +} From b3d3b9760b97e3e0b79f7d8011a8168cacf882d5 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 10 Oct 2024 08:01:31 +0000 Subject: [PATCH 19/47] chore: update and migrate E2E tests to v1 API --- test/pkg/debug/events.go | 34 +- test/pkg/debug/monitor.go | 21 +- test/pkg/debug/node.go | 33 +- test/pkg/debug/nodeclaim.go | 34 +- test/pkg/debug/pod.go | 28 +- test/pkg/debug/setup.go | 2 +- test/pkg/environment/azure/environment.go | 60 +-- test/pkg/environment/common/environment.go | 114 +++-- test/pkg/environment/common/expectations.go | 519 ++++++++++++++------ test/pkg/environment/common/monitor.go | 74 ++- test/pkg/environment/common/setup.go | 61 ++- test/suites/acr/suite_test.go | 4 +- test/suites/chaos/suite_test.go | 107 ++-- test/suites/drift/suite_test.go | 47 +- test/suites/gpu/suite_test.go | 6 +- test/suites/integration/daemonset_test.go | 3 - test/suites/integration/emptiness_test.go | 18 +- test/suites/integration/expiration_test.go | 116 ++--- test/suites/integration/suite_test.go | 4 +- test/suites/nodeclaim/nodeclaim_test.go | 40 +- test/suites/nodeclaim/suite_test.go | 4 +- test/suites/utilization/suite_test.go | 6 +- 22 files changed, 764 insertions(+), 571 deletions(-) diff --git a/test/pkg/debug/events.go b/test/pkg/debug/events.go index 80c2ef423..02997544a 100644 --- a/test/pkg/debug/events.go +++ b/test/pkg/debug/events.go @@ -24,7 +24,7 @@ import ( "github.com/samber/lo" "go.uber.org/multierr" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "sigs.k8s.io/controller-runtime/pkg/client" @@ -44,31 +44,19 @@ func NewEventClient(kubeClient client.Client) *EventClient { func (c *EventClient) DumpEvents(ctx context.Context) error { return multierr.Combine( - c.dumpKarpenterEvents(ctx), c.dumpPodEvents(ctx), c.dumpNodeEvents(ctx), ) } -func (c *EventClient) dumpKarpenterEvents(ctx context.Context) error { - el := &v1.EventList{} - if err := c.kubeClient.List(ctx, el, client.InNamespace("karpenter")); err != nil { - return err - } - for k, v := range collateEvents(filterTestEvents(el.Items, c.start)) { - fmt.Print(getEventInformation(k, v)) - } - return nil -} - func (c *EventClient) dumpPodEvents(ctx context.Context) error { - el := &v1.EventList{} + el := &corev1.EventList{} if err := c.kubeClient.List(ctx, el, &client.ListOptions{ FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Pod"}), }); err != nil { return err } - events := lo.Filter(filterTestEvents(el.Items, c.start), func(e v1.Event, _ int) bool { + events := lo.Filter(filterTestEvents(el.Items, c.start), func(e corev1.Event, _ int) bool { return e.InvolvedObject.Namespace != "kube-system" }) for k, v := range collateEvents(events) { @@ -78,7 +66,7 @@ func (c *EventClient) dumpPodEvents(ctx context.Context) error { } func (c *EventClient) dumpNodeEvents(ctx context.Context) error { - el := &v1.EventList{} + el := &corev1.EventList{} if err := c.kubeClient.List(ctx, el, &client.ListOptions{ FieldSelector: fields.SelectorFromSet(map[string]string{"involvedObject.kind": "Node"}), }); err != nil { @@ -90,8 +78,8 @@ func (c *EventClient) dumpNodeEvents(ctx context.Context) error { return nil } -func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event { - return lo.Filter(events, func(e v1.Event, _ int) bool { +func filterTestEvents(events []corev1.Event, startTime time.Time) []corev1.Event { + return lo.Filter(events, func(e corev1.Event, _ int) bool { if !e.EventTime.IsZero() { if e.EventTime.BeforeTime(&metav1.Time{Time: startTime}) { return false @@ -103,13 +91,13 @@ func filterTestEvents(events []v1.Event, startTime time.Time) []v1.Event { }) } -func collateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList { - eventMap := map[v1.ObjectReference]*v1.EventList{} +func collateEvents(events []corev1.Event) map[corev1.ObjectReference]*corev1.EventList { + eventMap := map[corev1.ObjectReference]*corev1.EventList{} for i := range events { elem := events[i] - objectKey := v1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name} + objectKey := corev1.ObjectReference{Kind: elem.InvolvedObject.Kind, Namespace: elem.InvolvedObject.Namespace, Name: elem.InvolvedObject.Name} if _, ok := eventMap[objectKey]; !ok { - eventMap[objectKey] = &v1.EventList{} + eventMap[objectKey] = &corev1.EventList{} } eventMap[objectKey].Items = append(eventMap[objectKey].Items, elem) } @@ -118,7 +106,7 @@ func collateEvents(events []v1.Event) map[v1.ObjectReference]*v1.EventList { // Partially copied from // https://github.com/kubernetes/kubernetes/blob/04ee339c7a4d36b4037ce3635993e2a9e395ebf3/staging/src/k8s.io/kubectl/pkg/describe/describe.go#L4232 -func getEventInformation(o v1.ObjectReference, el *v1.EventList) string { +func getEventInformation(o corev1.ObjectReference, el *corev1.EventList) string { sb := strings.Builder{} sb.WriteString(fmt.Sprintf("------- %s/%s%s EVENTS -------\n", strings.ToLower(o.Kind), lo.Ternary(o.Namespace != "", o.Namespace+"/", ""), o.Name)) diff --git a/test/pkg/debug/monitor.go b/test/pkg/debug/monitor.go index 3fe7466aa..461b08244 100644 --- a/test/pkg/debug/monitor.go +++ b/test/pkg/debug/monitor.go @@ -20,18 +20,14 @@ import ( "context" "sync" - "github.com/go-logr/zapr" + "github.com/awslabs/operatorpkg/controller" "github.com/samber/lo" "k8s.io/client-go/rest" - "knative.dev/pkg/logging" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - ctrl "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/metrics/server" - - "sigs.k8s.io/karpenter/pkg/operator/controller" - "sigs.k8s.io/karpenter/pkg/operator/scheme" ) type Monitor struct { @@ -42,22 +38,14 @@ type Monitor struct { } func New(ctx context.Context, config *rest.Config, kubeClient client.Client) *Monitor { - logger := logging.FromContext(ctx) - ctrl.SetLogger(zapr.NewLogger(logger.Desugar())) + log.SetLogger(log.FromContext(ctx)) mgr := lo.Must(controllerruntime.NewManager(config, controllerruntime.Options{ - Scheme: scheme.Scheme, - BaseContext: func() context.Context { - ctx := context.Background() - ctx = logging.WithLogger(ctx, logger) - logger.WithOptions() - return ctx - }, Metrics: server.Options{ BindAddress: "0", }, })) for _, c := range newControllers(kubeClient) { - lo.Must0(c.Builder(ctx, mgr).Complete(c), "failed to register controller") + lo.Must0(c.Register(ctx, mgr), "failed to register controller") } ctx, cancel := context.WithCancel(ctx) // this context is only meant for monitor start/stop return &Monitor{ @@ -84,7 +72,6 @@ func (m *Monitor) Stop() { func newControllers(kubeClient client.Client) []controller.Controller { return []controller.Controller{ - // NewMachineController(kubeClient), NewNodeClaimController(kubeClient), NewNodeController(kubeClient), NewPodController(kubeClient), diff --git a/test/pkg/debug/node.go b/test/pkg/debug/node.go index 2c2edc1e9..385628eed 100644 --- a/test/pkg/debug/node.go +++ b/test/pkg/debug/node.go @@ -21,7 +21,7 @@ import ( "fmt" "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -31,8 +31,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" ) @@ -46,12 +46,8 @@ func NewNodeController(kubeClient client.Client) *NodeController { } } -func (c *NodeController) Name() string { - return "node" -} - func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - n := &v1.Node{} + n := &corev1.Node{} if err := c.kubeClient.Get(ctx, req.NamespacedName, n); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] NODE %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -62,26 +58,27 @@ func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) ( return reconcile.Result{}, nil } -func (c *NodeController) GetInfo(ctx context.Context, n *v1.Node) string { +func (c *NodeController) GetInfo(ctx context.Context, n *corev1.Node) string { pods, _ := nodeutils.GetPods(ctx, c.kubeClient, n) - return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, v1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[v1beta1.NodeInitializedLabelKey], len(pods), n.Spec.Taints) + return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, corev1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[karpv1.NodeInitializedLabelKey], len(pods), n.Spec.Taints) } -func (c *NodeController) Builder(ctx context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1.Node{}). +func (c *NodeController) Register(ctx context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("node"). + For(&corev1.Node{}). WithEventFilter(predicate.And( predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldNode := e.ObjectOld.(*v1.Node) - newNode := e.ObjectNew.(*v1.Node) + oldNode := e.ObjectOld.(*corev1.Node) + newNode := e.ObjectNew.(*corev1.Node) return c.GetInfo(ctx, oldNode) != c.GetInfo(ctx, newNode) }, }, predicate.NewPredicateFuncs(func(o client.Object) bool { - return o.GetLabels()[v1beta1.NodePoolLabelKey] != "" + return o.GetLabels()[karpv1.NodePoolLabelKey] != "" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/debug/nodeclaim.go b/test/pkg/debug/nodeclaim.go index e7ae9f05f..635fb0268 100644 --- a/test/pkg/debug/nodeclaim.go +++ b/test/pkg/debug/nodeclaim.go @@ -30,8 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) type NodeClaimController struct { @@ -44,12 +43,8 @@ func NewNodeClaimController(kubeClient client.Client) *NodeClaimController { } } -func (c *NodeClaimController) Name() string { - return "nodeclaim" -} - func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - nc := &corev1beta1.NodeClaim{} + nc := &karpv1.NodeClaim{} if err := c.kubeClient.Get(ctx, req.NamespacedName, nc); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] NODECLAIM %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -60,25 +55,26 @@ func (c *NodeClaimController) Reconcile(ctx context.Context, req reconcile.Reque return reconcile.Result{}, nil } -func (c *NodeClaimController) GetInfo(nc *corev1beta1.NodeClaim) string { +func (c *NodeClaimController) GetInfo(nc *karpv1.NodeClaim) string { return fmt.Sprintf("ready=%t launched=%t registered=%t initialized=%t", - nc.StatusConditions().IsHappy(), - nc.StatusConditions().GetCondition(corev1beta1.Launched).IsTrue(), - nc.StatusConditions().GetCondition(corev1beta1.Registered).IsTrue(), - nc.StatusConditions().GetCondition(corev1beta1.Initialized).IsTrue(), + nc.StatusConditions().Root().IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeLaunched).IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeRegistered).IsTrue(), + nc.StatusConditions().Get(karpv1.ConditionTypeInitialized).IsTrue(), ) } -func (c *NodeClaimController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&corev1beta1.NodeClaim{}). +func (c *NodeClaimController) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclaim"). + For(&karpv1.NodeClaim{}). WithEventFilter(predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldNodeClaim := e.ObjectOld.(*corev1beta1.NodeClaim) - newNodeClaim := e.ObjectNew.(*corev1beta1.NodeClaim) + oldNodeClaim := e.ObjectOld.(*karpv1.NodeClaim) + newNodeClaim := e.ObjectNew.(*karpv1.NodeClaim) return c.GetInfo(oldNodeClaim) != c.GetInfo(newNodeClaim) }, }). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/debug/pod.go b/test/pkg/debug/pod.go index afdefaf9c..2f27a7d41 100644 --- a/test/pkg/debug/pod.go +++ b/test/pkg/debug/pod.go @@ -23,7 +23,7 @@ import ( "time" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -33,7 +33,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" "sigs.k8s.io/karpenter/pkg/utils/pod" ) @@ -47,12 +46,8 @@ func NewPodController(kubeClient client.Client) *PodController { } } -func (c *PodController) Name() string { - return "pod" -} - func (c *PodController) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - p := &v1.Pod{} + p := &corev1.Pod{} if err := c.kubeClient.Get(ctx, req.NamespacedName, p); err != nil { if errors.IsNotFound(err) { fmt.Printf("[DELETED %s] POD %s\n", time.Now().Format(time.RFC3339), req.NamespacedName.String()) @@ -63,7 +58,7 @@ func (c *PodController) Reconcile(ctx context.Context, req reconcile.Request) (r return reconcile.Result{}, nil } -func (c *PodController) GetInfo(p *v1.Pod) string { +func (c *PodController) GetInfo(p *corev1.Pod) string { var containerInfo strings.Builder for _, c := range p.Status.ContainerStatuses { if containerInfo.Len() > 0 { @@ -75,21 +70,22 @@ func (c *PodController) GetInfo(p *v1.Pod) string { pod.IsProvisionable(p), p.Status.Phase, p.Spec.NodeName, p.OwnerReferences, containerInfo.String()) } -func (c *PodController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.Adapt(controllerruntime. - NewControllerManagedBy(m). - For(&v1.Pod{}). +func (c *PodController) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("pod"). + For(&corev1.Pod{}). WithEventFilter(predicate.And( predicate.Funcs{ UpdateFunc: func(e event.UpdateEvent) bool { - oldPod := e.ObjectOld.(*v1.Pod) - newPod := e.ObjectNew.(*v1.Pod) + oldPod := e.ObjectOld.(*corev1.Pod) + newPod := e.ObjectNew.(*corev1.Pod) return c.GetInfo(oldPod) != c.GetInfo(newPod) }, }, predicate.NewPredicateFuncs(func(o client.Object) bool { - return o.GetNamespace() != "kube-system" && o.GetNamespace() != "karpenter" + return o.GetNamespace() != "kube-system" }), )). - WithOptions(controller.Options{MaxConcurrentReconciles: 10})) + WithOptions(controller.Options{MaxConcurrentReconciles: 10}). + Complete(c) } diff --git a/test/pkg/debug/setup.go b/test/pkg/debug/setup.go index b03671ebc..211beb565 100644 --- a/test/pkg/debug/setup.go +++ b/test/pkg/debug/setup.go @@ -24,7 +24,7 @@ import ( "k8s.io/client-go/rest" "sigs.k8s.io/controller-runtime/pkg/client" - . "github.com/onsi/gomega" //nolint:revive,stylecheck + . "github.com/onsi/gomega" ) const ( diff --git a/test/pkg/environment/azure/environment.go b/test/pkg/environment/azure/environment.go index f5797b2f6..6c8b1af18 100644 --- a/test/pkg/environment/azure/environment.go +++ b/test/pkg/environment/azure/environment.go @@ -21,20 +21,16 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - "k8s.io/client-go/kubernetes/scheme" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - coretest "sigs.k8s.io/karpenter/pkg/test" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "github.com/Azure/karpenter-provider-azure/pkg/apis" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/test" "github.com/Azure/karpenter-provider-azure/test/pkg/environment/common" ) func init() { - lo.Must0(apis.AddToScheme(scheme.Scheme)) - corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": v1.LabelTopologyZone}) + // TODO: should have core1beta1.NormalizedLabels too? + karpv1.NormalizedLabels = lo.Assign(karpv1.NormalizedLabels, map[string]string{"topology.disk.csi.azure.com/zone": v1.LabelTopologyZone}) } const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" @@ -53,56 +49,6 @@ func NewEnvironment(t *testing.T) *Environment { } } -func (env *Environment) DefaultNodePool(nodeClass *v1alpha2.AKSNodeClass) *corev1beta1.NodePool { - nodePool := coretest.NodePool() - nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, - } - nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ - {NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, - }}, - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, - }}, - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, - }}, - { - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1alpha2.LabelSKUFamily, - Operator: v1.NodeSelectorOpIn, - Values: []string{"D"}, - }}, - } - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} - nodePool.Spec.Disruption.ExpireAfter.Duration = nil - nodePool.Spec.Limits = corev1beta1.Limits(v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100"), - v1.ResourceMemory: resource.MustParse("1000Gi"), - }) - return nodePool -} - -func (env *Environment) ArmNodepool(nodeClass *v1alpha2.AKSNodeClass) *corev1beta1.NodePool { - nodePool := env.DefaultNodePool(nodeClass) - coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, - }}) - return nodePool -} - func (env *Environment) DefaultAKSNodeClass() *v1alpha2.AKSNodeClass { nodeClass := test.AKSNodeClass() return nodeClass diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index 86707a594..67cb5fe49 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -25,23 +25,26 @@ import ( "testing" "time" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/awslabs/operatorpkg/object" "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" "k8s.io/client-go/kubernetes" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" - loggingtesting "knative.dev/pkg/logging/testing" + + . "sigs.k8s.io/karpenter/pkg/utils/testing" //nolint:stylecheck + "knative.dev/pkg/system" controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" - "github.com/Azure/karpenter-provider-azure/pkg/apis" - coreapis "sigs.k8s.io/karpenter/pkg/apis" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/operator" + coretest "sigs.k8s.io/karpenter/pkg/test" ) type ContextKey string @@ -63,17 +66,17 @@ type Environment struct { } func NewEnvironment(t *testing.T) *Environment { - ctx := loggingtesting.TestContextWithLogger(t) + ctx := TestContextWithLogger(t) ctx, cancel := context.WithCancel(ctx) config := NewConfig() client := NewClient(ctx, config) - lo.Must0(os.Setenv(system.NamespaceEnvKey, "karpenter")) + lo.Must0(os.Setenv(system.NamespaceEnvKey, "kube-system")) if val, ok := os.LookupEnv("GIT_REF"); ok { ctx = context.WithValue(ctx, GitRefContextKey, val) } - gomega.SetDefaultEventuallyTimeout(5 * time.Minute) + gomega.SetDefaultEventuallyTimeout(16 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) return &Environment{ Context: ctx, @@ -98,32 +101,28 @@ func NewConfig() *rest.Config { } func NewClient(ctx context.Context, config *rest.Config) client.Client { - scheme := runtime.NewScheme() - lo.Must0(clientgoscheme.AddToScheme(scheme)) - lo.Must0(apis.AddToScheme(scheme)) - lo.Must0(coreapis.AddToScheme(scheme)) - - cache := lo.Must(cache.New(config, cache.Options{Scheme: scheme})) - lo.Must0(cache.IndexField(ctx, &v1.Pod{}, "spec.nodeName", func(o client.Object) []string { - pod := o.(*v1.Pod) + cache := lo.Must(cache.New(config, cache.Options{Scheme: scheme.Scheme})) + lo.Must0(cache.IndexField(ctx, &corev1.Pod{}, "spec.nodeName", func(o client.Object) []string { + pod := o.(*corev1.Pod) return []string{pod.Spec.NodeName} })) - lo.Must0(cache.IndexField(ctx, &v1.Event{}, "involvedObject.kind", func(o client.Object) []string { - evt := o.(*v1.Event) + lo.Must0(cache.IndexField(ctx, &corev1.Event{}, "involvedObject.kind", func(o client.Object) []string { + evt := o.(*corev1.Event) return []string{evt.InvolvedObject.Kind} })) - lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.unschedulable", func(o client.Object) []string { - node := o.(*v1.Node) + lo.Must0(cache.IndexField(ctx, &corev1.Node{}, "spec.unschedulable", func(o client.Object) []string { + node := o.(*corev1.Node) return []string{strconv.FormatBool(node.Spec.Unschedulable)} })) - lo.Must0(cache.IndexField(ctx, &v1.Node{}, "spec.taints[*].karpenter.sh/disruption", func(o client.Object) []string { - node := o.(*v1.Node) - t, _ := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return t.Key == corev1beta1.DisruptionTaintKey + lo.Must0(cache.IndexField(ctx, &corev1.Node{}, "spec.taints[*].karpenter.sh/disrupted", func(o client.Object) []string { + node := o.(*corev1.Node) + _, found := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return t.Key == karpv1.DisruptedTaintKey }) - return []string{t.Value} + return []string{lo.Ternary(found, "true", "false")} })) - c := lo.Must(client.New(config, client.Options{Scheme: scheme, Cache: &client.CacheOptions{Reader: cache}})) + + c := lo.Must(client.New(config, client.Options{Scheme: scheme.Scheme, Cache: &client.CacheOptions{Reader: cache}})) go func() { lo.Must0(cache.Start(ctx)) @@ -133,3 +132,62 @@ func NewClient(ctx context.Context, config *rest.Config) client.Client { } return c } + +func (env *Environment) DefaultNodePool(nodeClass *v1alpha2.AKSNodeClass) *karpv1.NodePool { + nodePool := coretest.NodePool() + nodePool.Spec.Template.Spec.NodeClassRef = &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + } + nodePool.Spec.Template.Spec.Requirements = []karpv1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelOSStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{string(corev1.Linux)}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeOnDemand}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureAmd64}, + }, + }, + { + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: v1alpha2.LabelSKUFamily, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"D"}, + }, + }, + } + + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("Never") + nodePool.Spec.Template.Spec.ExpireAfter.Duration = nil + nodePool.Spec.Limits = karpv1.Limits(corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1000"), // TODO: do we need that much? + corev1.ResourceMemory: resource.MustParse("1000Gi"), + }) + return nodePool +} + +func (env *Environment) ArmNodepool(nodeClass *v1alpha2.AKSNodeClass) *karpv1.NodePool { + nodePool := env.DefaultNodePool(nodeClass) + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelArchStable, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.ArchitectureArm64}, + }}) + return nodePool +} diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index 0b32b6aab..754d1eb5f 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -21,6 +21,7 @@ import ( "encoding/base64" "fmt" "io" + "math" "strings" "time" @@ -29,7 +30,7 @@ import ( "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" coordinationv1 "k8s.io/api/coordination/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,14 +38,15 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/transport" - "knative.dev/pkg/logging" - "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + "sigs.k8s.io/controller-runtime/pkg/log" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" pscheduling "sigs.k8s.io/karpenter/pkg/controllers/provisioning/scheduling" "sigs.k8s.io/karpenter/pkg/scheduling" "sigs.k8s.io/karpenter/pkg/test" + coreresources "sigs.k8s.io/karpenter/pkg/utils/resources" ) func (env *Environment) ExpectCreated(objects ...client.Object) { @@ -63,23 +65,38 @@ func (env *Environment) ExpectDeleted(objects ...client.Object) { GinkgoHelper() for _, object := range objects { Eventually(func(g Gomega) { - g.Expect(client.IgnoreNotFound(env.Client.Delete(env, object, client.PropagationPolicy(metav1.DeletePropagationForeground), &client.DeleteOptions{GracePeriodSeconds: ptr.Int64(0)}))).To(Succeed()) + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, object, client.PropagationPolicy(metav1.DeletePropagationForeground), &client.DeleteOptions{GracePeriodSeconds: lo.ToPtr(int64(0))}))).To(Succeed()) }).WithTimeout(time.Second * 10).Should(Succeed()) } } +// ExpectUpdated will update objects in the cluster to match the inputs. +// WARNING: This ignores the resource version check, which can result in +// overwriting changes made by other controllers in the cluster. +// This is useful in ensuring that we can clean up resources by patching +// out finalizers. +// Grab the object before making the updates to reduce the chance of this race. func (env *Environment) ExpectUpdated(objects ...client.Object) { GinkgoHelper() for _, o := range objects { Eventually(func(g Gomega) { current := o.DeepCopyObject().(client.Object) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(current), current)).To(Succeed()) + if current.GetResourceVersion() != o.GetResourceVersion() { + log.FromContext(env).Info(fmt.Sprintf("detected an update to an object (%s) with an outdated resource version, did you get the latest version of the object before patching?", lo.Must(apiutil.GVKForObject(o, env.Client.Scheme())))) + } o.SetResourceVersion(current.GetResourceVersion()) g.Expect(env.Client.Update(env.Context, o)).To(Succeed()) }).WithTimeout(time.Second * 10).Should(Succeed()) } } +// ExpectCreatedOrUpdated can update objects in the cluster to match the inputs. +// WARNING: ExpectUpdated ignores the resource version check, which can result in +// overwriting changes made by other controllers in the cluster. +// This is useful in ensuring that we can clean up resources by patching +// out finalizers. +// Grab the object before making the updates to reduce the chance of this race. func (env *Environment) ExpectCreatedOrUpdated(objects ...client.Object) { GinkgoHelper() for _, o := range objects { @@ -97,22 +114,22 @@ func (env *Environment) ExpectCreatedOrUpdated(objects ...client.Object) { } } -func (env *Environment) ExpectSettings() (res []v1.EnvVar) { +func (env *Environment) ExpectSettings() (res []corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) - return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) v1.EnvVar { + return lo.Map(d.Spec.Template.Spec.Containers[0].Env, func(v corev1.EnvVar, _ int) corev1.EnvVar { return *v.DeepCopy() }) } -func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsReplaced(vars ...corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) stored := d.DeepCopy() @@ -120,21 +137,21 @@ func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { if !equality.Semantic.DeepEqual(d, stored) { By("replacing environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } -func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsOverridden(vars ...corev1.EnvVar) { GinkgoHelper() d := &appsv1.Deployment{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) stored := d.DeepCopy() for _, v := range vars { - if _, i, ok := lo.FindIndexOf(d.Spec.Template.Spec.Containers[0].Env, func(e v1.EnvVar) bool { + if _, i, ok := lo.FindIndexOf(d.Spec.Template.Spec.Containers[0].Env, func(e corev1.EnvVar) bool { return e.Name == v.Name }); ok { d.Spec.Template.Spec.Containers[0].Env[i] = v @@ -144,41 +161,41 @@ func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { } if !equality.Semantic.DeepEqual(d, stored) { By("overriding environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } -func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) { +func (env *Environment) ExpectSettingsRemoved(vars ...corev1.EnvVar) { GinkgoHelper() - varNames := sets.New[string](lo.Map(vars, func(v v1.EnvVar, _ int) string { return v.Name })...) + varNames := sets.New(lo.Map(vars, func(v corev1.EnvVar, _ int) string { return v.Name })...) d := &appsv1.Deployment{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "karpenter", Name: "karpenter"}, d)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Namespace: "kube-system", Name: "karpenter"}, d)).To(Succeed()) Expect(d.Spec.Template.Spec.Containers).To(HaveLen(1)) stored := d.DeepCopy() - d.Spec.Template.Spec.Containers[0].Env = lo.Reject(d.Spec.Template.Spec.Containers[0].Env, func(v v1.EnvVar, _ int) bool { + d.Spec.Template.Spec.Containers[0].Env = lo.Reject(d.Spec.Template.Spec.Containers[0].Env, func(v corev1.EnvVar, _ int) bool { return varNames.Has(v.Name) }) if !equality.Semantic.DeepEqual(d, stored) { By("removing environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } -func (env *Environment) ExpectConfigMapExists(key types.NamespacedName) *v1.ConfigMap { +func (env *Environment) ExpectConfigMapExists(key types.NamespacedName) *corev1.ConfigMap { GinkgoHelper() - cm := &v1.ConfigMap{} + cm := &corev1.ConfigMap{} Expect(env.Client.Get(env, key, cm)).To(Succeed()) return cm } func (env *Environment) ExpectConfigMapDataReplaced(key types.NamespacedName, data ...map[string]string) (changed bool) { GinkgoHelper() - cm := &v1.ConfigMap{ + cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, @@ -201,7 +218,7 @@ func (env *Environment) ExpectConfigMapDataReplaced(key types.NamespacedName, da func (env *Environment) ExpectConfigMapDataOverridden(key types.NamespacedName, data ...map[string]string) (changed bool) { GinkgoHelper() - cm := &v1.ConfigMap{ + cm := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{ Name: key.Name, Namespace: key.Namespace, @@ -210,47 +227,93 @@ func (env *Environment) ExpectConfigMapDataOverridden(key types.NamespacedName, err := env.Client.Get(env, key, cm) Expect(client.IgnoreNotFound(err)).ToNot(HaveOccurred()) - stored := cm.DeepCopy() cm.Data = lo.Assign(append([]map[string]string{cm.Data}, data...)...) - // If the data hasn't changed, we can just return and not update anything - if equality.Semantic.DeepEqual(stored, cm) { - return false - } // Update the configMap to update the settings env.ExpectCreatedOrUpdated(cm) return true } -func (env *Environment) ExpectExists(obj client.Object) { +func (env *Environment) ExpectExists(obj client.Object) client.Object { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) }).WithTimeout(time.Second * 5).Should(Succeed()) + return obj +} + +func (env *Environment) EventuallyExpectBound(pods ...*corev1.Pod) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.Spec.NodeName).ToNot(BeEmpty()) + } + }).Should(Succeed()) } -func (env *Environment) EventuallyExpectHealthy(pods ...*v1.Pod) { +func (env *Environment) EventuallyExpectHealthy(pods ...*corev1.Pod) { GinkgoHelper() env.EventuallyExpectHealthyWithTimeout(-1, pods...) } -func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*v1.Pod) { +func (env *Environment) EventuallyExpectTerminating(pods ...*corev1.Pod) { GinkgoHelper() - for _, pod := range pods { - Eventually(func(g Gomega) { + env.EventuallyExpectTerminatingWithTimeout(-1, pods...) +} + +func (env *Environment) EventuallyExpectTerminatingWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, pod := range pods { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) g.Expect(pod.Status.Conditions).To(ContainElement(And( - HaveField("Type", Equal(v1.PodReady)), - HaveField("Status", Equal(v1.ConditionTrue)), + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), ))) - }).WithTimeout(timeout).Should(Succeed()) - } + } + }).WithTimeout(timeout).Should(Succeed()) +} + +func (env *Environment) ConsistentlyExpectTerminatingPods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be terminating for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }, duration.String()).Should(Succeed()) +} + +func (env *Environment) ConsistentlyExpectHealthyPods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be ready for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.Status.Conditions).To(ContainElement(And( + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), + ))) + } + }, duration.String()).Should(Succeed()) } func (env *Environment) EventuallyExpectKarpenterRestarted() { GinkgoHelper() By("rolling out the new karpenter deployment") - env.EventuallyExpectRollout("karpenter", "karpenter") + env.EventuallyExpectRollout("karpenter", "kube-system") env.ExpectKarpenterLeaseOwnerChanged() } @@ -261,7 +324,7 @@ func (env *Environment) ExpectKarpenterLeaseOwnerChanged() { pods := env.ExpectKarpenterPods() Eventually(func(g Gomega) { name := env.ExpectActiveKarpenterPodName() - g.Expect(lo.ContainsBy(pods, func(p *v1.Pod) bool { + g.Expect(lo.ContainsBy(pods, func(p *corev1.Pod) bool { return p.Name == name })).To(BeTrue()) }).Should(Succeed()) @@ -278,39 +341,39 @@ func (env *Environment) EventuallyExpectRollout(name, namespace string) { "kubectl.kubernetes.io/restartedAt": time.Now().Format(time.RFC3339), } deploy.Spec.Template.Annotations = lo.Assign(deploy.Spec.Template.Annotations, restartedAtAnnotation) - Expect(env.Client.Patch(env.Context, deploy, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, deploy, client.StrategicMergeFrom(stored))).To(Succeed()) By("waiting for the newly generated deployment to rollout") Eventually(func(g Gomega) { - podList := &v1.PodList{} + podList := &corev1.PodList{} g.Expect(env.Client.List(env.Context, podList, client.InNamespace(namespace))).To(Succeed()) - pods := lo.Filter(podList.Items, func(p v1.Pod, _ int) bool { + pods := lo.Filter(podList.Items, func(p corev1.Pod, _ int) bool { return p.Annotations["kubectl.kubernetes.io/restartedAt"] == restartedAtAnnotation["kubectl.kubernetes.io/restartedAt"] }) g.Expect(len(pods)).To(BeNumerically("==", lo.FromPtr(deploy.Spec.Replicas))) for _, pod := range pods { g.Expect(pod.Status.Conditions).To(ContainElement(And( - HaveField("Type", Equal(v1.PodReady)), - HaveField("Status", Equal(v1.ConditionTrue)), + HaveField("Type", Equal(corev1.PodReady)), + HaveField("Status", Equal(corev1.ConditionTrue)), ))) - g.Expect(pod.Status.Phase).To(Equal(v1.PodRunning)) + g.Expect(pod.Status.Phase).To(Equal(corev1.PodRunning)) } }).Should(Succeed()) } -func (env *Environment) ExpectKarpenterPods() []*v1.Pod { +func (env *Environment) ExpectKarpenterPods() []*corev1.Pod { GinkgoHelper() - podList := &v1.PodList{} + podList := &corev1.PodList{} Expect(env.Client.List(env.Context, podList, client.MatchingLabels{ "app.kubernetes.io/instance": "karpenter", })).To(Succeed()) - return lo.Map(podList.Items, func(p v1.Pod, _ int) *v1.Pod { return &p }) + return lo.Map(podList.Items, func(p corev1.Pod, _ int) *corev1.Pod { return &p }) } func (env *Environment) ExpectActiveKarpenterPodName() string { GinkgoHelper() lease := &coordinationv1.Lease{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Name: "karpenter-leader-election", Namespace: "karpenter"}, lease)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: "karpenter-leader-election", Namespace: "kube-system"}, lease)).To(Succeed()) // Holder identity for lease is always in the format "_ holderArr := strings.Split(lo.FromPtr(lease.Spec.HolderIdentity), "_") @@ -319,12 +382,12 @@ func (env *Environment) ExpectActiveKarpenterPodName() string { return holderArr[0] } -func (env *Environment) ExpectActiveKarpenterPod() *v1.Pod { +func (env *Environment) ExpectActiveKarpenterPod() *corev1.Pod { GinkgoHelper() podName := env.ExpectActiveKarpenterPodName() - pod := &v1.Pod{} - Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "karpenter"}, pod)).To(Succeed()) + pod := &corev1.Pod{} + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: podName, Namespace: "kube-system"}, pod)).To(Succeed()) return pod } @@ -335,23 +398,43 @@ func (env *Environment) EventuallyExpectPendingPodCount(selector labels.Selector }).Should(Succeed()) } -func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector, numPods int) { +func (env *Environment) EventuallyExpectBoundPodCount(selector labels.Selector, numPods int) []*corev1.Pod { + GinkgoHelper() + var res []*corev1.Pod + Eventually(func(g Gomega) { + res = []*corev1.Pod{} + podList := &corev1.PodList{} + g.Expect(env.Client.List(env.Context, podList, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) + for i := range podList.Items { + if podList.Items[i].Spec.NodeName != "" { + res = append(res, &podList.Items[i]) + } + } + g.Expect(res).To(HaveLen(numPods)) + }).Should(Succeed()) + return res +} + +func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector, numPods int) []*corev1.Pod { By(fmt.Sprintf("waiting for %d pods matching selector %s to be ready", numPods, selector.String())) GinkgoHelper() - env.EventuallyExpectHealthyPodCountWithTimeout(-1, selector, numPods) + return env.EventuallyExpectHealthyPodCountWithTimeout(-1, selector, numPods) } -func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) { +func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) []*corev1.Pod { GinkgoHelper() + var pods []*corev1.Pod Eventually(func(g Gomega) { - g.Expect(env.Monitor.RunningPodsCount(selector)).To(Equal(numPods)) + pods = env.Monitor.RunningPods(selector) + g.Expect(pods).To(HaveLen(numPods)) }).WithTimeout(timeout).Should(Succeed()) + return pods } -func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []*v1.Pod { +func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []*corev1.Pod { GinkgoHelper() - podList := &v1.PodList{} + podList := &corev1.PodList{} Expect(env.Client.List(env.Context, podList, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) return lo.ToSlicePtr(podList.Items) } @@ -391,7 +474,7 @@ func (env *Environment) EventuallyExpectNotFoundAssertion(objects ...client.Obje }) } -func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() createdNodes := env.Monitor.CreatedNodes() Expect(len(createdNodes)).To(BeNumerically(comparator, count), @@ -399,74 +482,117 @@ func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []* return createdNodes } -func NodeNames(nodes []*v1.Node) []string { - return lo.Map(nodes, func(n *v1.Node, index int) string { +func (env *Environment) ExpectNodeCount(comparator string, count int) []*corev1.Node { + GinkgoHelper() + + nodeList := &corev1.NodeList{} + Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + Expect(len(nodeList.Items)).To(BeNumerically(comparator, count)) + return lo.ToSlicePtr(nodeList.Items) +} + +func (env *Environment) ExpectNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { + GinkgoHelper() + + nodeClaimList := &karpv1.NodeClaimList{} + Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) + return lo.ToSlicePtr(nodeClaimList.Items) +} + +func NodeClaimNames(nodeClaims []*karpv1.NodeClaim) []string { + return lo.Map(nodeClaims, func(n *karpv1.NodeClaim, index int) string { + return n.Name + }) +} + +func NodeNames(nodes []*corev1.Node) []string { + return lo.Map(nodes, func(n *corev1.Node, index int) string { return n.Name }) } -func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration string) []*v1.Node { +func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration time.Duration) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("expecting nodes to be %s to %d for %s", comparator, count, duration)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Consistently(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), fmt.Sprintf("expected %d nodes, had %d (%v) for %s", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)), duration)) - }, duration).Should(Succeed()) + }, duration.String()).Should(Succeed()) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectCordonedNodeCountLegacy(comparator string, count int) []*v1.Node { +func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) (taintedNodes []*corev1.Node) { GinkgoHelper() - By(fmt.Sprintf("waiting for cordoned nodes to be %s to %d", comparator, count)) - nodeList := &v1.NodeList{} - Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.unschedulable": "true"})).To(Succeed()) - g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), - fmt.Sprintf("expected %d cordoned nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) - }).Should(Succeed()) - return lo.ToSlicePtr(nodeList.Items) + return env.ConsistentlyExpectDisruptionsWithNodeCount(0, nodeCount, duration) } -func (env *Environment) EventuallyExpectNodesUncordonedLegacyWithTimeout(timeout time.Duration, nodes ...*v1.Node) { +// ConsistentlyExpectDisruptionsWithNodeCount will continually ensure that there are exactly disruptingNodes with totalNodes (including replacements and existing nodes) +func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNodes, totalNodes int, duration time.Duration) (taintedNodes []*corev1.Node) { GinkgoHelper() - By(fmt.Sprintf("waiting for %d nodes to be uncordoned", len(nodes))) - nodeList := &v1.NodeList{} - Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.unschedulable": "true"})).To(Succeed()) - cordonedNodeNames := lo.Map(nodeList.Items, func(n v1.Node, _ int) string { return n.Name }) - g.Expect(cordonedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *v1.Node, _ int) interface{} { return n.Name })...)) - }).WithTimeout(timeout).Should(Succeed()) + nodes := []corev1.Node{} + Consistently(func(g Gomega) { + // Ensure we don't change our NodeClaims + nodeClaimList := &karpv1.NodeClaimList{} + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeClaimList.Items).To(HaveLen(totalNodes)) + + nodeList := &corev1.NodeList{} + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(totalNodes)) + + nodes = lo.Filter(nodeList.Items, func(n corev1.Node, _ int) bool { + _, ok := lo.Find(n.Spec.Taints, func(t corev1.Taint) bool { + return karpv1.IsDisruptingTaint(t) + }) + return ok + }) + g.Expect(nodes).To(HaveLen(disruptingNodes)) + }, duration).Should(Succeed()) + return lo.ToSlicePtr(nodes) } -func (env *Environment) EventuallyExpectCordonedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() - By(fmt.Sprintf("waiting for cordoned nodes to be %s to %d", comparator, count)) - nodeList := &v1.NodeList{} + By(fmt.Sprintf("waiting for tainted nodes to be %s to %d", comparator, count)) + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disrupted": "true"})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), - fmt.Sprintf("expected %d cordoned nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) + fmt.Sprintf("expected %d tainted nodes, had %d (%v)", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)))) }).Should(Succeed()) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectNodesUncordonedWithTimeout(timeout time.Duration, nodes ...*v1.Node) { +func (env *Environment) EventuallyExpectNodesUntaintedWithTimeout(timeout time.Duration, nodes ...*corev1.Node) { GinkgoHelper() - By(fmt.Sprintf("waiting for %d nodes to be uncordoned", len(nodes))) - nodeList := &v1.NodeList{} + By(fmt.Sprintf("waiting for %d nodes to be untainted", len(nodes))) + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { - g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disruption": "disrupting"})).To(Succeed()) - cordonedNodeNames := lo.Map(nodeList.Items, func(n v1.Node, _ int) string { return n.Name }) - g.Expect(cordonedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *v1.Node, _ int) interface{} { return n.Name })...)) + g.Expect(env.Client.List(env, nodeList, client.MatchingFields{"spec.taints[*].karpenter.sh/disrupted": "true"})).To(Succeed()) + taintedNodeNames := lo.Map(nodeList.Items, func(n corev1.Node, _ int) string { return n.Name }) + g.Expect(taintedNodeNames).ToNot(ContainElements(lo.Map(nodes, func(n *corev1.Node, _ int) interface{} { return n.Name })...)) }).WithTimeout(timeout).Should(Succeed()) } -func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { GinkgoHelper() By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) - nodeList := &v1.NodeList{} + nodeClaimList := &karpv1.NodeClaimList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d nodeclaims, had %d (%v)", count, len(nodeClaimList.Items), NodeClaimNames(lo.ToSlicePtr(nodeClaimList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeClaimList.Items) +} + +func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*corev1.Node { + GinkgoHelper() + By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), @@ -475,10 +601,10 @@ func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { +func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for nodes with selector %v to be %s to %d", selector, comparator, count)) - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel}, client.MatchingLabelsSelector{Selector: selector})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), @@ -487,10 +613,10 @@ func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for created nodes to be %s to %d", comparator, count)) - var createdNodes []*v1.Node + var createdNodes []*corev1.Node Eventually(func(g Gomega) { createdNodes = env.Monitor.CreatedNodes() g.Expect(len(createdNodes)).To(BeNumerically(comparator, count), @@ -499,10 +625,10 @@ func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, coun return createdNodes } -func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for deleted nodes to be %s to %d", comparator, count)) - var deletedNodes []*v1.Node + var deletedNodes []*corev1.Node Eventually(func(g Gomega) { deletedNodes = env.Monitor.DeletedNodes() g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), @@ -511,13 +637,13 @@ func (env *Environment) EventuallyExpectDeletedNodeCount(comparator string, coun return deletedNodes } -func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*v1.Node { +func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator string, count int, selector labels.Selector) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for deleted nodes with selector %v to be %s to %d", selector, comparator, count)) - var deletedNodes []*v1.Node + var deletedNodes []*corev1.Node Eventually(func(g Gomega) { deletedNodes = env.Monitor.DeletedNodes() - deletedNodes = lo.Filter(deletedNodes, func(n *v1.Node, _ int) bool { + deletedNodes = lo.Filter(deletedNodes, func(n *corev1.Node, _ int) bool { return selector.Matches(labels.Set(n.Labels)) }) g.Expect(len(deletedNodes)).To(BeNumerically(comparator, count), @@ -526,63 +652,97 @@ func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator return deletedNodes } -func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*v1.Node { +func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*corev1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for initialized nodes to be %s to %d", comparator, count)) - var nodes []*v1.Node + var nodes []*corev1.Node Eventually(func(g Gomega) { nodes = env.Monitor.CreatedNodes() - nodes = lo.Filter(nodes, func(n *v1.Node, _ int) bool { - return n.Labels[corev1beta1.NodeInitializedLabelKey] == "true" + nodes = lo.Filter(nodes, func(n *corev1.Node, _ int) bool { + return n.Labels[karpv1.NodeInitializedLabelKey] == "true" }) g.Expect(len(nodes)).To(BeNumerically(comparator, count)) }).Should(Succeed()) return nodes } -func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim { +func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, count int) []*karpv1.NodeClaim { GinkgoHelper() By(fmt.Sprintf("waiting for created nodeclaims to be %s to %d", comparator, count)) - nodeClaimList := &corev1beta1.NodeClaimList{} + nodeClaimList := &karpv1.NodeClaimList{} Eventually(func(g Gomega) { g.Expect(env.Client.List(env.Context, nodeClaimList)).To(Succeed()) g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) }).Should(Succeed()) - return lo.Map(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) *corev1beta1.NodeClaim { + return lo.Map(nodeClaimList.Items, func(nc karpv1.NodeClaim, _ int) *karpv1.NodeClaim { return &nc }) } -func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) { +func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*karpv1.NodeClaim) { + GinkgoHelper() Eventually(func(g Gomega) { for _, nc := range nodeClaims { - temp := &corev1beta1.NodeClaim{} + temp := &karpv1.NodeClaim{} g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nc), temp)).Should(Succeed()) - g.Expect(temp.StatusConditions().IsHappy()).To(BeTrue()) + g.Expect(temp.StatusConditions().Root().IsTrue()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*karpv1.NodeClaim) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeDrifted).IsTrue()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) ConsistentlyExpectNodeClaimsNotDrifted(duration time.Duration, nodeClaims ...*karpv1.NodeClaim) { + GinkgoHelper() + nodeClaimNames := lo.Map(nodeClaims, func(nc *karpv1.NodeClaim, _ int) string { return nc.Name }) + By(fmt.Sprintf("consistently expect nodeclaims %s not to be drifted for %s", nodeClaimNames, duration)) + Consistently(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeDrifted)).To(BeNil()) + } + }, duration).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectConsolidatable(nodeClaims ...*karpv1.NodeClaim) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().Get(karpv1.ConditionTypeConsolidatable).IsTrue()).To(BeTrue()) } }).Should(Succeed()) } -func (env *Environment) GetNode(nodeName string) v1.Node { +func (env *Environment) GetNode(nodeName string) corev1.Node { GinkgoHelper() - var node v1.Node + var node corev1.Node Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) return node } func (env *Environment) ExpectNoCrashes() { GinkgoHelper() - _, crashed := lo.Find(lo.Values(env.Monitor.RestartCount()), func(restartCount int) bool { - return restartCount > 0 - }) - Expect(crashed).To(BeFalse(), "expected karpenter containers to not crash") + for k, v := range env.Monitor.RestartCount("kube-system") { + if strings.Contains(k, "karpenter") && v > 0 { + Fail("expected karpenter containers to not crash") + } + } } var ( lastLogged = metav1.Now() ) -func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { +func (env *Environment) printControllerLogs(options *corev1.PodLogOptions) { fmt.Println("------- START CONTROLLER LOGS -------") defer fmt.Println("------- END CONTROLLER LOGS -------") @@ -595,34 +755,34 @@ func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { temp := options.DeepCopy() // local version of the log options fmt.Printf("------- pod/%s -------\n", pod.Name) - if pod.Status.ContainerStatuses[0].RestartCount > 0 { + if len(pod.Status.ContainerStatuses) > 0 && pod.Status.ContainerStatuses[0].RestartCount > 0 { fmt.Printf("[PREVIOUS CONTAINER LOGS]\n") temp.Previous = true } - stream, err := env.KubeClient.CoreV1().Pods("karpenter").GetLogs(pod.Name, temp).Stream(env.Context) + stream, err := env.KubeClient.CoreV1().Pods("kube-system").GetLogs(pod.Name, temp).Stream(env.Context) if err != nil { - logging.FromContext(env.Context).Errorf("fetching controller logs: %s", err) + log.FromContext(env.Context).Error(err, "failed fetching controller logs") return } - log := &bytes.Buffer{} - _, err = io.Copy(log, stream) + raw := &bytes.Buffer{} + _, err = io.Copy(raw, stream) Expect(err).ToNot(HaveOccurred()) - logging.FromContext(env.Context).Info(log) + log.FromContext(env.Context).Info(raw.String()) } } -func (env *Environment) EventuallyExpectMinUtilization(resource v1.ResourceName, comparator string, value float64) { +func (env *Environment) EventuallyExpectMinUtilization(resource corev1.ResourceName, comparator string, value float64) { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Monitor.MinUtilization(resource)).To(BeNumerically(comparator, value)) }).Should(Succeed()) } -func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName, comparator string, value float64) { +func (env *Environment) EventuallyExpectAvgUtilization(resource corev1.ResourceName, comparator string, value float64) { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value)) - }, 10*time.Minute).Should(Succeed()) + }, 12*time.Minute).Should(Succeed()) } func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.ObjectKey, name, value string, containers ...string) { @@ -633,7 +793,7 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj Expect(len(ds.Spec.Template.Spec.Containers)).To(BeNumerically("==", 1)) containers = append(containers, ds.Spec.Template.Spec.Containers[0].Name) } - patch := client.MergeFrom(ds.DeepCopy()) + patch := client.StrategicMergeFrom(ds.DeepCopy()) containerNames := sets.New(containers...) for ci := range ds.Spec.Template.Spec.Containers { c := &ds.Spec.Template.Spec.Containers[ci] @@ -641,17 +801,77 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj continue } // If the env var already exists, update its value. Otherwise, create a new var. - if _, i, ok := lo.FindIndexOf(c.Env, func(e v1.EnvVar) bool { + if _, i, ok := lo.FindIndexOf(c.Env, func(e corev1.EnvVar) bool { return e.Name == name }); ok { c.Env[i].Value = value } else { - c.Env = append(c.Env, v1.EnvVar{Name: name, Value: value}) + c.Env = append(c.Env, corev1.EnvVar{Name: name, Value: value}) } } Expect(env.Client.Patch(env.Context, ds, patch)).To(Succeed()) } +// ForcePodsToSpread ensures that currently scheduled pods get spread evenly across all passed nodes by deleting pods off of existing +// nodes and waiting them to reschedule. This is useful for scenarios where you want to force the nodes be underutilized +// but you want to keep a consistent count of nodes rather than leaving around empty ones. +func (env *Environment) ForcePodsToSpread(nodes ...*corev1.Node) { + GinkgoHelper() + + // Get the total count of pods across + podCount := 0 + for _, n := range nodes { + podCount += len(env.ExpectActivePodsForNode(n.Name)) + } + maxPodsPerNode := int(math.Ceil(float64(podCount) / float64(len(nodes)))) + + By(fmt.Sprintf("forcing %d pods to spread across %d nodes", podCount, len(nodes))) + start := time.Now() + for { + var nodePods []*corev1.Pod + node, found := lo.Find(nodes, func(n *corev1.Node) bool { + nodePods = env.ExpectActivePodsForNode(n.Name) + return len(nodePods) > maxPodsPerNode + }) + if !found { + break + } + // Set the nodes to unschedulable so that the pods won't reschedule. + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) + stored := node.DeepCopy() + node.Spec.Unschedulable = true + Expect(env.Client.Patch(env.Context, node, client.StrategicMergeFrom(stored))).To(Succeed()) + for _, pod := range nodePods[maxPodsPerNode:] { + env.ExpectDeleted(pod) + } + Eventually(func(g Gomega) { + g.Expect(len(env.ExpectActivePodsForNode(node.Name))).To(Or(Equal(maxPodsPerNode), Equal(maxPodsPerNode-1))) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // TODO: Consider moving this time check to an Eventually poll. This gets a little tricker with helper functions + // since you need to make sure that your Expectation helper functions are scoped to to your "g Gomega" scope + // so that you don't fail the first time you get a failure on your expectation + if time.Since(start) > time.Minute*15 { + Fail("forcing pods to spread failed due to a timeout") + } + } + for _, n := range nodes { + stored := n.DeepCopy() + n.Spec.Unschedulable = false + Expect(env.Client.Patch(env.Context, n, client.StrategicMergeFrom(stored))).To(Succeed()) + } +} + +func (env *Environment) ExpectActivePodsForNode(nodeName string) []*corev1.Pod { + GinkgoHelper() + podList := &corev1.PodList{} + Expect(env.Client.List(env, podList, client.MatchingFields{"spec.nodeName": nodeName}, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + + return lo.Filter(lo.ToSlicePtr(podList.Items), func(p *corev1.Pod, _ int) bool { + return p.DeletionTimestamp.IsZero() + }) +} + func (env *Environment) ExpectCABundle() string { // Discover CA Bundle from the REST client. We could alternatively // have used the simpler client-go InClusterConfig() method. @@ -662,11 +882,11 @@ func (env *Environment) ExpectCABundle() string { Expect(err).ToNot(HaveOccurred()) _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! Expect(err).ToNot(HaveOccurred()) - logging.FromContext(env.Context).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) + log.FromContext(env.Context).WithValues("length", len(transportConfig.TLS.CAData)).V(1).Info("discovered caBundle") return base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData) } -func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { +func (env *Environment) GetDaemonSetCount(np *karpv1.NodePool) int { GinkgoHelper() // Performs the same logic as the scheduler to get the number of daemonset @@ -675,14 +895,35 @@ func (env *Environment) GetDaemonSetCount(np *corev1beta1.NodePool) int { Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) return lo.CountBy(daemonSetList.Items, func(d appsv1.DaemonSet) bool { - p := &v1.Pod{Spec: d.Spec.Template.Spec} - nodeTemplate := pscheduling.NewNodeClaimTemplate(np) - if err := scheduling.Taints(nodeTemplate.Spec.Taints).Tolerates(p); err != nil { + p := &corev1.Pod{Spec: d.Spec.Template.Spec} + nodeClaimTemplate := pscheduling.NewNodeClaimTemplate(np) + if err := scheduling.Taints(nodeClaimTemplate.Spec.Taints).Tolerates(p); err != nil { return false } - if err := nodeTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabels); err != nil { + if err := nodeClaimTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabels); err != nil { return false } return true }) } + +func (env *Environment) GetDaemonSetOverhead(np *karpv1.NodePool) corev1.ResourceList { + GinkgoHelper() + + // Performs the same logic as the scheduler to get the number of daemonset + // pods that we estimate we will need to schedule as overhead to each node + daemonSetList := &appsv1.DaemonSetList{} + Expect(env.Client.List(env.Context, daemonSetList)).To(Succeed()) + + return coreresources.RequestsForPods(lo.FilterMap(daemonSetList.Items, func(ds appsv1.DaemonSet, _ int) (*corev1.Pod, bool) { + p := &corev1.Pod{Spec: ds.Spec.Template.Spec} + nodeClaimTemplate := pscheduling.NewNodeClaimTemplate(np) + if err := scheduling.Taints(nodeClaimTemplate.Spec.Taints).Tolerates(p); err != nil { + return nil, false + } + if err := nodeClaimTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p), scheduling.AllowUndefinedWellKnownLabels); err != nil { + return nil, false + } + return p, true + })...) +} diff --git a/test/pkg/environment/common/monitor.go b/test/pkg/environment/common/monitor.go index 5c3ef6add..46be9e0a1 100644 --- a/test/pkg/environment/common/monitor.go +++ b/test/pkg/environment/common/monitor.go @@ -22,15 +22,15 @@ import ( "math" "sync" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" - "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" "github.com/samber/lo" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/utils/resources" ) @@ -41,21 +41,21 @@ type Monitor struct { mu sync.RWMutex - nodesAtReset map[string]*v1.Node + nodesAtReset map[string]*corev1.Node } type state struct { - pods v1.PodList - nodes map[string]*v1.Node // node name -> node - nodePods map[string][]*v1.Pod // node name -> pods bound to the node - nodeRequests map[string]v1.ResourceList // node name -> sum of pod resource requests + pods corev1.PodList + nodes map[string]*corev1.Node // node name -> node + nodePods map[string][]*corev1.Pod // node name -> pods bound to the node + nodeRequests map[string]corev1.ResourceList // node name -> sum of pod resource requests } func NewMonitor(ctx context.Context, kubeClient client.Client) *Monitor { m := &Monitor{ ctx: ctx, kubeClient: kubeClient, - nodesAtReset: map[string]*v1.Node{}, + nodesAtReset: map[string]*corev1.Node{}, } m.Reset() return m @@ -72,14 +72,14 @@ func (m *Monitor) Reset() { // RestartCount returns the containers and number of restarts for that container for all containers in the pods in the // given namespace -func (m *Monitor) RestartCount() map[string]int { +func (m *Monitor) RestartCount(namespace string) map[string]int { st := m.poll() m.mu.RLock() defer m.mu.RUnlock() restarts := map[string]int{} for _, pod := range st.pods.Items { - if pod.Namespace != "karpenter" { + if pod.Namespace != namespace { continue } for _, cs := range pod.Status.ContainerStatuses { @@ -107,36 +107,35 @@ func (m *Monitor) CreatedNodeCount() int { } // NodesAtReset returns a slice of nodes that the monitor saw at the last reset -func (m *Monitor) NodesAtReset() []*v1.Node { +func (m *Monitor) NodesAtReset() []*corev1.Node { m.mu.RLock() defer m.mu.RUnlock() return deepCopySlice(lo.Values(m.nodesAtReset)) } // Nodes returns all the nodes on the cluster -func (m *Monitor) Nodes() []*v1.Node { +func (m *Monitor) Nodes() []*corev1.Node { st := m.poll() return lo.Values(st.nodes) } // CreatedNodes returns the nodes that have been created since the last reset (essentially Nodes - NodesAtReset) -func (m *Monitor) CreatedNodes() []*v1.Node { - resetNodeNames := sets.NewString(lo.Map(m.NodesAtReset(), func(n *v1.Node, _ int) string { return n.Name })...) - return lo.Filter(m.Nodes(), func(n *v1.Node, _ int) bool { return !resetNodeNames.Has(n.Name) }) +func (m *Monitor) CreatedNodes() []*corev1.Node { + resetNodeNames := sets.NewString(lo.Map(m.NodesAtReset(), func(n *corev1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.Nodes(), func(n *corev1.Node, _ int) bool { return !resetNodeNames.Has(n.Name) }) } // DeletedNodes returns the nodes that have been deleted since the last reset (essentially NodesAtReset - Nodes) -func (m *Monitor) DeletedNodes() []*v1.Node { - currentNodeNames := sets.NewString(lo.Map(m.Nodes(), func(n *v1.Node, _ int) string { return n.Name })...) - return lo.Filter(m.NodesAtReset(), func(n *v1.Node, _ int) bool { return !currentNodeNames.Has(n.Name) }) +func (m *Monitor) DeletedNodes() []*corev1.Node { + currentNodeNames := sets.NewString(lo.Map(m.Nodes(), func(n *corev1.Node, _ int) string { return n.Name })...) + return lo.Filter(m.NodesAtReset(), func(n *corev1.Node, _ int) bool { return !currentNodeNames.Has(n.Name) }) } // PendingPods returns the number of pending pods matching the given selector -func (m *Monitor) PendingPods(selector labels.Selector) []*v1.Pod { - var pods []*v1.Pod +func (m *Monitor) PendingPods(selector labels.Selector) []*corev1.Pod { + var pods []*corev1.Pod for _, pod := range m.poll().pods.Items { - pod := pod - if pod.Status.Phase != v1.PodPending { + if pod.Status.Phase != corev1.PodPending { continue } if selector.Matches(labels.Set(pod.Labels)) { @@ -151,11 +150,10 @@ func (m *Monitor) PendingPodsCount(selector labels.Selector) int { } // RunningPods returns the number of running pods matching the given selector -func (m *Monitor) RunningPods(selector labels.Selector) []*v1.Pod { - var pods []*v1.Pod +func (m *Monitor) RunningPods(selector labels.Selector) []*corev1.Pod { + var pods []*corev1.Pod for _, pod := range m.poll().pods.Items { - pod := pod - if pod.Status.Phase != v1.PodRunning { + if pod.Status.Phase != corev1.PodRunning { continue } if selector.Matches(labels.Set(pod.Labels)) { @@ -170,19 +168,19 @@ func (m *Monitor) RunningPodsCount(selector labels.Selector) int { } func (m *Monitor) poll() state { - var nodes v1.NodeList + var nodes corev1.NodeList if err := m.kubeClient.List(m.ctx, &nodes); err != nil { - logging.FromContext(m.ctx).Errorf("listing nodes, %s", err) + log.FromContext(m.ctx).Error(err, "failed listing nodes") } - var pods v1.PodList + var pods corev1.PodList if err := m.kubeClient.List(m.ctx, &pods); err != nil { - logging.FromContext(m.ctx).Errorf("listing pods, %s", err) + log.FromContext(m.ctx).Error(err, "failing listing pods") } st := state{ - nodes: map[string]*v1.Node{}, + nodes: map[string]*corev1.Node{}, pods: pods, - nodePods: map[string][]*v1.Pod{}, - nodeRequests: map[string]v1.ResourceList{}, + nodePods: map[string][]*corev1.Pod{}, + nodeRequests: map[string]corev1.ResourceList{}, } for i := range nodes.Items { st.nodes[nodes.Items[i].Name] = &nodes.Items[i] @@ -202,7 +200,7 @@ func (m *Monitor) poll() state { return st } -func (m *Monitor) AvgUtilization(resource v1.ResourceName) float64 { +func (m *Monitor) AvgUtilization(resource corev1.ResourceName) float64 { utilization := m.nodeUtilization(resource) sum := 0.0 for _, v := range utilization { @@ -211,7 +209,7 @@ func (m *Monitor) AvgUtilization(resource v1.ResourceName) float64 { return sum / float64(len(utilization)) } -func (m *Monitor) MinUtilization(resource v1.ResourceName) float64 { +func (m *Monitor) MinUtilization(resource corev1.ResourceName) float64 { min := math.MaxFloat64 for _, v := range m.nodeUtilization(resource) { min = math.Min(v, min) @@ -219,13 +217,13 @@ func (m *Monitor) MinUtilization(resource v1.ResourceName) float64 { return min } -func (m *Monitor) nodeUtilization(resource v1.ResourceName) []float64 { +func (m *Monitor) nodeUtilization(resource corev1.ResourceName) []float64 { st := m.poll() var utilization []float64 for nodeName, requests := range st.nodeRequests { allocatable := st.nodes[nodeName].Status.Allocatable[resource] // skip any nodes we didn't launch - if st.nodes[nodeName].Labels[v1beta1.NodePoolLabelKey] == "" { + if st.nodes[nodeName].Labels[karpv1.NodePoolLabelKey] == "" { continue } if allocatable.IsZero() { diff --git a/test/pkg/environment/common/setup.go b/test/pkg/environment/common/setup.go index 6d33fe50e..b56f71117 100644 --- a/test/pkg/environment/common/setup.go +++ b/test/pkg/environment/common/setup.go @@ -27,34 +27,39 @@ import ( . "github.com/onsi/gomega" //nolint:revive,stylecheck "github.com/samber/lo" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "sigs.k8s.io/karpenter/pkg/utils/pod" ) +const TestingFinalizer = "testing/finalizer" + var ( CleanableObjects = []client.Object{ - &v1.Pod{}, + &corev1.Pod{}, &appsv1.Deployment{}, + &appsv1.StatefulSet{}, &appsv1.DaemonSet{}, &policyv1.PodDisruptionBudget{}, - &v1.PersistentVolumeClaim{}, - &v1.PersistentVolume{}, + &corev1.PersistentVolumeClaim{}, + &corev1.PersistentVolume{}, &storagev1.StorageClass{}, - &corev1beta1.NodePool{}, - &v1.LimitRange{}, + &karpv1.NodePool{}, + &corev1.LimitRange{}, &schedulingv1.PriorityClass{}, - &v1.Node{}, - &corev1beta1.NodeClaim{}, + &corev1.Node{}, + &karpv1.NodeClaim{}, + &v1alpha2.AKSNodeClass{}, } ) @@ -70,28 +75,28 @@ func (env *Environment) BeforeEach() { } func (env *Environment) ExpectCleanCluster() { - var nodes v1.NodeList + var nodes corev1.NodeList Expect(env.Client.List(env.Context, &nodes)).To(Succeed()) for _, node := range nodes.Items { if len(node.Spec.Taints) == 0 && !node.Spec.Unschedulable { Fail(fmt.Sprintf("expected system pool node %s to be tainted", node.Name)) } } - var pods v1.PodList + var pods corev1.PodList Expect(env.Client.List(env.Context, &pods)).To(Succeed()) for i := range pods.Items { Expect(pods.Items[i].Namespace).ToNot(Equal("default"), fmt.Sprintf("expected no pods in the `default` namespace, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name)) } Eventually(func(g Gomega) { - var pods v1.PodList + var pods corev1.PodList g.Expect(env.Client.List(env.Context, &pods)).To(Succeed()) for i := range pods.Items { g.Expect(pod.IsProvisionable(&pods.Items[i])).To(BeFalse(), fmt.Sprintf("expected to have no provisionable pods, found %s/%s", pods.Items[i].Namespace, pods.Items[i].Name)) } }).WithPolling(10 * time.Second).WithTimeout(5 * time.Minute).Should(Succeed()) - for _, obj := range []client.Object{&corev1beta1.NodePool{}, &v1alpha2.AKSNodeClass{}} { + for _, obj := range []client.Object{&karpv1.NodePool{}, &v1alpha2.AKSNodeClass{}} { metaList := &metav1.PartialObjectMetadataList{} gvk := lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme())) metaList.SetGroupVersionKind(gvk) @@ -108,7 +113,7 @@ func (env *Environment) Cleanup() { func (env *Environment) AfterEach() { debug.AfterEach(env.Context) - env.printControllerLogs(&v1.PodLogOptions{Container: "controller"}) + env.printControllerLogs(&corev1.PodLogOptions{Container: "controller"}) } func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { @@ -128,7 +133,10 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { // are deleting so that we avoid getting client-side throttled workqueue.ParallelizeUntil(env, 50, len(metaList.Items), func(i int) { defer GinkgoRecover() - g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], client.PropagationPolicy(metav1.DeletePropagationForeground)))).To(Succeed()) + g.Expect(env.ExpectTestingFinalizerRemoved(&metaList.Items[i])).To(Succeed()) + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], + client.PropagationPolicy(metav1.DeletePropagationForeground), + &client.DeleteOptions{GracePeriodSeconds: lo.ToPtr(int64(0))}))).To(Succeed()) }) // If the deletes eventually succeed, we should have no elements here at the end of the test g.Expect(env.Client.List(env, metaList, client.HasLabels([]string{test.DiscoveryLabel}), client.Limit(1))).To(Succeed()) @@ -138,3 +146,26 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { } wg.Wait() } + +func (env *Environment) ExpectTestingFinalizerRemoved(obj client.Object) error { + metaObj := &metav1.PartialObjectMetadata{} + metaObj.SetGroupVersionKind(lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))) + if err := env.Client.Get(env, client.ObjectKeyFromObject(obj), metaObj); err != nil { + return client.IgnoreNotFound(err) + } + deepCopy := metaObj.DeepCopy() + metaObj.Finalizers = lo.Reject(metaObj.Finalizers, func(finalizer string, _ int) bool { + return finalizer == TestingFinalizer + }) + + if !equality.Semantic.DeepEqual(metaObj, deepCopy) { + // If the Group is the "core" APIs, then we can strategic merge patch + // CRDs do not currently have support for strategic merge patching, so we can't blindly do it + // https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#advanced-features-and-flexibility:~:text=Yes-,strategic%2Dmerge%2Dpatch,-The%20new%20endpoints + if metaObj.GroupVersionKind().Group == "" { + return client.IgnoreNotFound(env.Client.Patch(env, metaObj, client.StrategicMergeFrom(deepCopy))) + } + return client.IgnoreNotFound(env.Client.Patch(env, metaObj, client.MergeFrom(deepCopy))) + } + return nil +} diff --git a/test/suites/acr/suite_test.go b/test/suites/acr/suite_test.go index 076b03ca0..9833dd6ff 100644 --- a/test/suites/acr/suite_test.go +++ b/test/suites/acr/suite_test.go @@ -30,13 +30,13 @@ import ( "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) var env *azure.Environment var nodeClass *v1alpha2.AKSNodeClass -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool var pauseImage string func TestAcr(t *testing.T) { diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index fe8ffedc5..fc00162b3 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package chaos +package chaos_test import ( "context" @@ -23,10 +23,8 @@ import ( "testing" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" "k8s.io/client-go/rest" @@ -37,14 +35,21 @@ import ( "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + coretest "sigs.k8s.io/karpenter/pkg/test" + nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" + + v1alpha2 "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/test/pkg/debug" "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/test" - nodeutils "sigs.k8s.io/karpenter/pkg/utils/node" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var env *azure.Environment +var nodeClass *v1alpha2.AKSNodeClass +var nodePool *karpv1.NodePool func TestChaos(t *testing.T) { RegisterFailHandler(Fail) @@ -54,7 +59,11 @@ func TestChaos(t *testing.T) { RunSpecs(t, "Chaos") } -var _ = BeforeEach(func() { env.BeforeEach() }) +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultAKSNodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) @@ -64,21 +73,20 @@ var _ = Describe("Chaos", func() { ctx, cancel := context.WithCancel(env.Context) defer cancel() - nodeClass := env.DefaultAKSNodeClass() - nodePool := env.DefaultNodePool(nodeClass) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, - }}) - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil + nodePool = coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: karpv1.CapacityTypeLabelKey, + Operator: corev1.NodeSelectorOpIn, + Values: []string{karpv1.CapacityTypeSpot}, + }, + }) + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") numPods := 1 - dep := test.Deployment(test.DeploymentOptions{ + dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: int32(numPods), - PodOptions: test.PodOptions{ + PodOptions: coretest.PodOptions{ Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, @@ -95,31 +103,21 @@ var _ = Describe("Chaos", func() { // Expect that we never get over a high number of nodes Consistently(func(g Gomega) { - list := &v1.NodeList{} - g.Expect(env.Client.List(env.Context, list, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + list := &corev1.NodeList{} + g.Expect(env.Client.List(env.Context, list, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) g.Expect(len(list.Items)).To(BeNumerically("<", 35)) }, time.Minute*5).Should(Succeed()) }) - It("should not produce a runaway scale-up when ttlSecondsAfterEmpty is enabled", Label(debug.NoWatch), Label(debug.NoEvents), func() { + It("should not produce a runaway scale-up when emptiness is enabled", Label(debug.NoWatch), Label(debug.NoEvents), func() { ctx, cancel := context.WithCancel(env.Context) defer cancel() - nodeClass := env.DefaultAKSNodeClass() - nodePool := env.DefaultNodePool(nodeClass) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, - }}) - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(30 * time.Second)} - + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("30s") numPods := 1 - dep := test.Deployment(test.DeploymentOptions{ + dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: int32(numPods), - PodOptions: test.PodOptions{ - Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", + PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "my-app"}, }, @@ -135,8 +133,8 @@ var _ = Describe("Chaos", func() { // Expect that we never get over a high number of nodes Consistently(func(g Gomega) { - list := &v1.NodeList{} - g.Expect(env.Client.List(env.Context, list, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + list := &corev1.NodeList{} + g.Expect(env.Client.List(env.Context, list, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) g.Expect(len(list.Items)).To(BeNumerically("<", 35)) }, time.Minute*5).Should(Succeed()) }) @@ -148,15 +146,15 @@ type taintAdder struct { } func (t *taintAdder) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { - node := &v1.Node{} + node := &corev1.Node{} if err := t.kubeClient.Get(ctx, req.NamespacedName, node); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) } - mergeFrom := client.MergeFrom(node.DeepCopy()) - taint := v1.Taint{ + mergeFrom := client.StrategicMergeFrom(node.DeepCopy()) + taint := corev1.Taint{ Key: "test", Value: "true", - Effect: v1.TaintEffectNoExecute, + Effect: corev1.TaintEffectNoExecute, } if !lo.Contains(node.Spec.Taints, taint) { node.Spec.Taints = append(node.Spec.Taints, taint) @@ -169,10 +167,12 @@ func (t *taintAdder) Reconcile(ctx context.Context, req reconcile.Request) (reco func (t *taintAdder) Builder(mgr manager.Manager) *controllerruntime.Builder { return controllerruntime.NewControllerManagedBy(mgr). - For(&v1.Node{}). + For(&corev1.Node{}). + // TODO: restore after controller-runtime version update + // WithOptions(controller.Options{SkipNameValidation: lo.ToPtr(true)}). WithEventFilter(predicate.NewPredicateFuncs(func(obj client.Object) bool { - node := obj.(*v1.Node) - if _, ok := node.Labels[test.DiscoveryLabel]; !ok { + node := obj.(*corev1.Node) + if _, ok := node.Labels[coretest.DiscoveryLabel]; !ok { return false } return true @@ -199,24 +199,23 @@ func startNodeCountMonitor(ctx context.Context, kubeClient client.Client) { deletedNodes := atomic.Int64{} factory := informers.NewSharedInformerFactoryWithOptions(env.KubeClient, time.Second*30, - informers.WithTweakListOptions(func(l *metav1.ListOptions) { l.LabelSelector = corev1beta1.NodePoolLabelKey })) + informers.WithTweakListOptions(func(l *metav1.ListOptions) { l.LabelSelector = karpv1.NodePoolLabelKey })) nodeInformer := factory.Core().V1().Nodes().Informer() - _, err := nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _ = lo.Must(nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: func(_ interface{}) { createdNodes.Add(1) }, DeleteFunc: func(_ interface{}) { deletedNodes.Add(1) }, - }) - Expect(err).ToNot(HaveOccurred()) + })) factory.Start(ctx.Done()) go func() { for { - list := &v1.NodeList{} - if err := kubeClient.List(ctx, list, client.HasLabels{test.DiscoveryLabel}); err == nil { - readyCount := lo.CountBy(list.Items, func(n v1.Node) bool { - return nodeutils.GetCondition(&n, v1.NodeReady).Status == v1.ConditionTrue + list := &corev1.NodeList{} + if err := kubeClient.List(ctx, list, client.HasLabels{coretest.DiscoveryLabel}); err == nil { + readyCount := lo.CountBy(list.Items, func(n corev1.Node) bool { + return nodeutils.GetCondition(&n, corev1.NodeReady).Status == corev1.ConditionTrue }) fmt.Printf("[NODE COUNT] CURRENT: %d | READY: %d | CREATED: %d | DELETED: %d\n", len(list.Items), readyCount, createdNodes.Load(), deletedNodes.Load()) } diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 1cd36a7ba..9e6b63861 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -23,22 +23,21 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" - "sigs.k8s.io/karpenter/pkg/test" + coretest "sigs.k8s.io/karpenter/pkg/test" ) var env *azure.Environment var nodeClass *v1alpha2.AKSNodeClass -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool func TestDrift(t *testing.T) { RegisterFailHandler(Fail) @@ -58,29 +57,32 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Drift", func() { - var pod *v1.Pod + var pod *corev1.Pod BeforeEach(func() { - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) + env.ExpectSettingsOverridden(corev1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, + coretest.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: corev1.NodeSelectorRequirement{ + Key: corev1.LabelInstanceTypeStable, + Operator: corev1.NodeSelectorOpIn, Values: []string{"Standard_DS2_v2"}, }}) // Add a do-not-disrupt pod so that we can check node metadata before we disrupt - pod = test.Pod(test.PodOptions{ + pod = coretest.Pod(coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + karpv1.DoNotDisruptAnnotationKey: "true", }, }, - ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("0.5")}}, + ResourceRequirements: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("0.5")}}, Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", }) }) + + // TODO: Add budget tests + It("should deprovision nodes that have drifted due to labels", func() { By(fmt.Sprintf("creating pod %s, nodepool %s, and nodeclass %s", pod.Name, nodePool.Name, nodeClass.Name)) @@ -100,14 +102,11 @@ var _ = Describe("Drift", func() { env.ExpectCreatedOrUpdated(nodePool) By(fmt.Sprintf("waiting for nodeclaim %s to be marked as drifted", nodeClaim.Name)) - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) - }).Should(Succeed()) + + env.EventuallyExpectDrifted(nodeClaim) By(fmt.Sprintf("waiting for pod %s to to update", pod.Name)) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) By(fmt.Sprintf("expect pod %s, nodeclaim %s, and node %s to eventually not exist", pod.Name, nodeClaim.Name, node.Name)) @@ -139,14 +138,10 @@ var _ = Describe("Drift", func() { env.ExpectCreatedOrUpdated(nodeClass) By(fmt.Sprintf("waiting for nodeclaim %s to be marked as drifted", nodeClaim.Name)) - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) - }).Should(Succeed()) + env.EventuallyExpectDrifted(nodeClaim) By(fmt.Sprintf("waiting for pod %s to to update", pod.Name)) - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) env.ExpectUpdated(pod) By(fmt.Sprintf("expect pod %s, nodeclaim %s, and node %s to eventually not exist", pod.Name, nodeClaim.Name, node.Name)) diff --git a/test/suites/gpu/suite_test.go b/test/suites/gpu/suite_test.go index f2d62add8..704c45fac 100644 --- a/test/suites/gpu/suite_test.go +++ b/test/suites/gpu/suite_test.go @@ -29,7 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" @@ -57,13 +57,13 @@ var _ = Describe("GPU", func() { nodePool := env.DefaultNodePool(nodeClass) // Relax default SKU family selector to allow for GPU nodes - test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1alpha2.LabelSKUFamily, Operator: v1.NodeSelectorOpExists, }}) // Exclude some of the more expensive GPU SKUs - nodePool.Spec.Limits = corev1beta1.Limits{ + nodePool.Spec.Limits = karpv1.Limits{ v1.ResourceCPU: resource.MustParse("25"), v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"), } diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index f8ce91de6..d4368c5b6 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -28,7 +28,6 @@ import ( . "github.com/onsi/gomega" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test" ) @@ -39,8 +38,6 @@ var _ = Describe("DaemonSet", func() { var dep *appsv1.Deployment BeforeEach(func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized - nodePool.Spec.Disruption.ConsolidateAfter = nil nodePool.Spec.Template.Labels = map[string]string{"testing/cluster": "test"} priorityclass = &schedulingv1.PriorityClass{ diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index 045578f10..303516c5f 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -17,9 +17,6 @@ limitations under the License. package integration_test import ( - "time" - - "github.com/samber/lo" "k8s.io/apimachinery/pkg/labels" "knative.dev/pkg/ptr" @@ -28,14 +25,15 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) var _ = Describe("Emptiness", func() { + // TODO: add budget tests It("should terminate an empty node", func() { - nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)} + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("10s") const numPods = 1 deployment := test.Deployment(test.DeploymentOptions{Replicas: numPods}) @@ -51,14 +49,10 @@ var _ = Describe("Emptiness", func() { deployment.Spec.Replicas = ptr.Int32(0) Expect(env.Client.Patch(env, deployment, client.MergeFrom(persisted))).To(Succeed()) - By("waiting for the nodeclaim emptiness status condition to propagate") - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue()) - }).Should(Succeed()) + env.EventuallyExpectConsolidatable(nodeClaim) By("waiting for the nodeclaim to deprovision when past its ConsolidateAfter timeout of 0") - nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") env.ExpectUpdated(nodePool) env.EventuallyExpectNotFound(nodeClaim, node) diff --git a/test/suites/integration/expiration_test.go b/test/suites/integration/expiration_test.go index 96dd51784..278038541 100644 --- a/test/suites/integration/expiration_test.go +++ b/test/suites/integration/expiration_test.go @@ -17,74 +17,65 @@ limitations under the License. package integration_test import ( - "time" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" - v1 "k8s.io/api/core/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" coretest "sigs.k8s.io/karpenter/pkg/test" ) var _ = Describe("Expiration", func() { - - var _ = BeforeEach(func() { - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} - }) - - It("should expire the node after the expiration is reached", func() { - var numPods int32 = 1 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int + BeforeEach(func() { + numPods = 1 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + Labels: map[string]string{ + "app": "my-app", }, - Labels: map[string]string{"app": "large-app"}, }, + TerminationGracePeriodSeconds: lo.ToPtr[int64](0), }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) + It("should expire the node after the expiration is reached", func() { + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("2m") env.ExpectCreated(nodeClass, nodePool, dep) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - // Expect that the NodeClaim will get an expired status condition - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) - }).Should(Succeed()) - - // Remove the do-not-disrupt annotation so that the Nodes are now deprovisionable - for _, pod := range env.ExpectPodsMatchingSelector(selector) { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - - // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) - _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return corev1beta1.IsDisruptingTaint(t) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return t.MatchTaint(&karpv1.DisruptedNoScheduleTaint) }) g.Expect(ok).To(BeTrue()) }).Should(Succeed()) - // Set the expireAfter to "Never" to make sure new node isn't deleted - // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and - // racing at the end of the E2E test, leaking node resources into subsequent tests - nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.EventuallyExpectCreatedNodeCount("==", 2) + // Set the limit to 0 to make sure we don't continue to create nodeClaims. + // This is CRITICAL since it prevents leaking node resources into subsequent tests + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("0"), + } env.ExpectUpdated(nodePool) // After the deletion timestamp is set and all pods are drained @@ -93,29 +84,22 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectCreatedNodeClaimCount("==", 1) env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should replace expired node with a single node and schedule all pods", func() { + // Set expire after to 5 minutes since we have to respect PDB and move over pods one at a time from one node to another. + // The new nodes should not expire before all the pods are moved over. + nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("5m") var numPods int32 = 5 // We should setup a PDB that will only allow a minimum of 1 pod to be pending at a time minAvailable := intstr.FromInt32(numPods - 1) pdb := coretest.PodDisruptionBudget(coretest.PDBOptions{ Labels: map[string]string{ - "app": "large-app", + "app": "my-app", }, MinAvailable: &minAvailable, }) - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - }, - }) + dep.Spec.Replicas = &numPods selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, pdb, dep) @@ -124,35 +108,21 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, int(numPods)) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration - // Set the expireAfter value to get the node deleted - nodePool.Spec.Disruption.ExpireAfter.Duration = lo.ToPtr(time.Minute) - env.ExpectUpdated(nodePool) - - // Expect that the NodeClaim will get an expired status condition - Eventually(func(g Gomega) { - g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) - g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) - }).Should(Succeed()) - - // Remove the do-not-disruption annotation so that the Nodes are now deprovisionable - for _, pod := range env.ExpectPodsMatchingSelector(selector) { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - - // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) - _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { - return corev1beta1.IsDisruptingTaint(t) + _, ok := lo.Find(node.Spec.Taints, func(t corev1.Taint) bool { + return t.MatchTaint(&karpv1.DisruptedNoScheduleTaint) }) g.Expect(ok).To(BeTrue()) }).Should(Succeed()) - // Set the expireAfter to "Never" to make sure new node isn't deleted - // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and - // racing at the end of the E2E test, leaking node resources into subsequent tests - nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.EventuallyExpectCreatedNodeCount("==", 2) + // Set the limit to 0 to make sure we don't continue to create nodeClaims. + // This is CRITICAL since it prevents leaking node resources into subsequent tests + nodePool.Spec.Limits = karpv1.Limits{ + corev1.ResourceCPU: resource.MustParse("0"), + } env.ExpectUpdated(nodePool) // After the deletion timestamp is set and all pods are drained diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index 310601726..32e382c58 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -24,12 +24,12 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) var env *azure.Environment var nodeClass *v1alpha2.AKSNodeClass -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool func TestIntegration(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/suites/nodeclaim/nodeclaim_test.go b/test/suites/nodeclaim/nodeclaim_test.go index abadb26a6..9b9b32d78 100644 --- a/test/suites/nodeclaim/nodeclaim_test.go +++ b/test/suites/nodeclaim/nodeclaim_test.go @@ -24,28 +24,28 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "sigs.k8s.io/karpenter/pkg/utils/resources" ) var _ = Describe("StandaloneNodeClaim", func() { It("should create a standard NodeClaim within the 'D' sku family", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ {NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1alpha2.LabelSKUFamily, Operator: v1.NodeSelectorOpIn, Values: []string{"D"}, }}, {NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, + Key: karpv1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + Values: []string{karpv1.CapacityTypeOnDemand}, }}, }, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -57,15 +57,15 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should create a standard NodeClaim based on resource requests", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Resources: corev1beta1.ResourceRequirements{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Resources: karpv1.ResourceRequirements{ Requests: v1.ResourceList{ v1.ResourceCPU: resource.MustParse("1"), v1.ResourceMemory: resource.MustParse("16Gi"), }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -77,7 +77,7 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should create a NodeClaim propagating all the NodeClaim spec details", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ "custom-annotation": "custom-value", @@ -86,7 +86,7 @@ var _ = Describe("StandaloneNodeClaim", func() { "custom-label": "custom-value", }, }, - Spec: corev1beta1.NodeClaimSpec{ + Spec: karpv1.NodeClaimSpec{ Taints: []v1.Taint{ { Key: "custom-taint", @@ -99,7 +99,7 @@ var _ = Describe("StandaloneNodeClaim", func() { Value: "other-custom-value", }, }, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, @@ -124,21 +124,21 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNodeClaimsReady(nodeClaim) }) It("should remove the cloudProvider NodeClaim when the cluster NodeClaim is deleted", func() { - nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ - Spec: corev1beta1.NodeClaimSpec{ - Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + nodeClaim := test.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + Requirements: []karpv1.NodeSelectorRequirementWithMinValues{ {NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1alpha2.LabelSKUFamily, Operator: v1.NodeSelectorOpIn, Values: []string{"D"}, }}, {NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, + Key: karpv1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + Values: []string{karpv1.CapacityTypeOnDemand}, }}, }, - NodeClassRef: &corev1beta1.NodeClassReference{ + NodeClassRef: &karpv1.NodeClassReference{ Name: nodeClass.Name, }, }, diff --git a/test/suites/nodeclaim/suite_test.go b/test/suites/nodeclaim/suite_test.go index 62c5668e6..c33e29294 100644 --- a/test/suites/nodeclaim/suite_test.go +++ b/test/suites/nodeclaim/suite_test.go @@ -24,12 +24,12 @@ import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/test/pkg/environment/azure" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" ) var env *azure.Environment var nodeClass *v1alpha2.AKSNodeClass -var nodePool *corev1beta1.NodePool +var nodePool *karpv1.NodePool func TestNodeClaim(t *testing.T) { RegisterFailHandler(Fail) diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go index 06357cb21..b114c3f8d 100644 --- a/test/suites/utilization/suite_test.go +++ b/test/suites/utilization/suite_test.go @@ -25,7 +25,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/labels" - "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" @@ -53,8 +53,8 @@ var _ = Describe("Utilization", func() { ubuntuNodeClassArm := env.DefaultAKSNodeClass() DescribeTable("should provision one pod per node", - func(nodeClass *v1alpha2.AKSNodeClass, nodePool *v1beta1.NodePool) { - test.ReplaceRequirements(nodePool, v1beta1.NodeSelectorRequirementWithMinValues{ + func(nodeClass *v1alpha2.AKSNodeClass, nodePool *karpv1.NodePool) { + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ NodeSelectorRequirement: v1.NodeSelectorRequirement{ Key: v1alpha2.LabelSKUCPU, Operator: v1.NodeSelectorOpLt, From 2f45040ad20cd7128d333c461e30653ee0df0d27 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:16:38 +0000 Subject: [PATCH 20/47] feat: refresh and relink CRDs --- .../templates/karpenter.azure.com_aksnodeclasses.yaml | 2 +- charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | 1 + charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | 1 + charts/karpenter/crds/karpenter.azure.com_aksnodeclasses.yaml | 1 + charts/karpenter/crds/karpenter.sh_nodeclaims.yaml | 1 + charts/karpenter/crds/karpenter.sh_nodepools.yaml | 1 + pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml | 2 +- 7 files changed, 7 insertions(+), 2 deletions(-) create mode 120000 charts/karpenter/crds/karpenter.azure.com_aksnodeclasses.yaml create mode 120000 charts/karpenter/crds/karpenter.sh_nodeclaims.yaml create mode 120000 charts/karpenter/crds/karpenter.sh_nodepools.yaml diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index 9b5afdeb9..bb925da03 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.3 + controller-gen.kubebuilder.io/version: v0.16.4 name: aksnodeclasses.karpenter.azure.com spec: group: karpenter.azure.com diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index 269ca1387..832863d03 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -827,3 +827,4 @@ spec: namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} port: {{ .Values.webhook.port }} {{- end }} + diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index f0330c96b..b838292b5 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -1078,3 +1078,4 @@ spec: namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} port: {{ .Values.webhook.port }} {{- end }} + diff --git a/charts/karpenter/crds/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter/crds/karpenter.azure.com_aksnodeclasses.yaml new file mode 120000 index 000000000..4917e92a9 --- /dev/null +++ b/charts/karpenter/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml new file mode 120000 index 000000000..3f572b575 --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_nodepools.yaml b/charts/karpenter/crds/karpenter.sh_nodepools.yaml new file mode 120000 index 000000000..36d2d1dd9 --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index 9b5afdeb9..bb925da03 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.16.3 + controller-gen.kubebuilder.io/version: v0.16.4 name: aksnodeclasses.karpenter.azure.com spec: group: karpenter.azure.com From 1da798c7140bd0bedf443d7bdef15f4666a05f11 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:36:23 +0000 Subject: [PATCH 21/47] fix: move code generation into subfolders to fix golangci-lint (typecheck detecting multiple main.go) --- .../main.go} | 0 hack/code/{prices_gen.go => prices_gen/main.go} | 0 hack/codegen.sh | 4 ++-- 3 files changed, 2 insertions(+), 2 deletions(-) rename hack/code/{instancetype_testdata_gen.go => instancetype_testdata_gen/main.go} (100%) rename hack/code/{prices_gen.go => prices_gen/main.go} (100%) diff --git a/hack/code/instancetype_testdata_gen.go b/hack/code/instancetype_testdata_gen/main.go similarity index 100% rename from hack/code/instancetype_testdata_gen.go rename to hack/code/instancetype_testdata_gen/main.go diff --git a/hack/code/prices_gen.go b/hack/code/prices_gen/main.go similarity index 100% rename from hack/code/prices_gen.go rename to hack/code/prices_gen/main.go diff --git a/hack/codegen.sh b/hack/codegen.sh index 664dd23d1..581be327d 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -12,7 +12,7 @@ pricing() { NO_UPDATE=$' pkg/providers/pricing/zz_generated.pricing.go | 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' SUBJECT="Pricing" - go run hack/code/prices_gen.go -- "${GENERATED_FILE}" + go run hack/code/prices_gen/main.go -- "${GENERATED_FILE}" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" @@ -32,7 +32,7 @@ skugen() { NO_UPDATE=" pkg/fake/zz_generated.sku.$location.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)" SUBJECT="SKUGEN" - go run hack/code/instancetype_testdata_gen.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_D16plds_v5,Standard_F16s_v2,Standard_NC6s,Standard_NC6s_v3,Standard_NC16as_T4_v3,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3" + go run hack/code/instancetype_testdata_gen/main.go -- "${GENERATED_FILE}" "$location" "Standard_B1s,Standard_A0,Standard_D2_v2,Standard_D2_v3,Standard_DS2_v2,Standard_D2s_v3,Standard_D2_v5,Standard_D16plds_v5,Standard_F16s_v2,Standard_NC6s,Standard_NC6s_v3,Standard_NC16as_T4_v3,Standard_NC24ads_A100_v4,Standard_M8-2ms,Standard_D4s_v3,Standard_D64s_v3,Standard_DC8s_v3" go fmt "${GENERATED_FILE}" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") From e390c261492622e542c3599666c70e4021d2e88b Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:43:59 +0000 Subject: [PATCH 22/47] fix: enable most of govet in golangci --- .golangci.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.golangci.yaml b/.golangci.yaml index b26e479f9..0cbc630c4 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -33,8 +33,9 @@ linters-settings: gocyclo: min-complexity: 11 govet: - enable: - - shadow + enable-all: true + disable: + - fieldalignment revive: rules: - name: dot-imports From 6a47c36407da6a11ff870aad009ad7b29ae8167f Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:45:25 +0000 Subject: [PATCH 23/47] fix(linting): exclude alt operator logger --- .golangci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yaml b/.golangci.yaml index 0cbc630c4..3049d52db 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -80,3 +80,5 @@ issues: - hack - charts - designs + exclude-files: + - pkg/alt/karpenter-core/pkg/operator/logger.go # copy From 3e5aeccaa6e5f14584d82ba275d4f2a04245cc9e Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 23:07:09 +0000 Subject: [PATCH 24/47] fix: add nodeclass termination controller --- .../nodeclass/termination/controller.go | 125 ++++++++++++++++++ .../nodeclass/termination/events.go | 38 ++++++ .../nodeclass/termination/suite_test.go | 124 +++++++++++++++++ pkg/test/aksnodeclass.go | 16 +++ pkg/utils/utils.go | 17 +++ 5 files changed, 320 insertions(+) create mode 100644 pkg/controllers/nodeclass/termination/controller.go create mode 100644 pkg/controllers/nodeclass/termination/events.go create mode 100644 pkg/controllers/nodeclass/termination/suite_test.go diff --git a/pkg/controllers/nodeclass/termination/controller.go b/pkg/controllers/nodeclass/termination/controller.go new file mode 100644 index 000000000..db0a6a0e5 --- /dev/null +++ b/pkg/controllers/nodeclass/termination/controller.go @@ -0,0 +1,125 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package termination + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/karpenter/pkg/operator/injection" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/types" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/awslabs/operatorpkg/reasonable" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/events" +) + +type Controller struct { + kubeClient client.Client + recorder events.Recorder +} + +func NewController(kubeClient client.Client, recorder events.Recorder) *Controller { + return &Controller{ + kubeClient: kubeClient, + recorder: recorder, + } +} + +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.termination") + + if !nodeClass.GetDeletionTimestamp().IsZero() { + return c.finalize(ctx, nodeClass) + } + return reconcile.Result{}, nil +} + +func (c *Controller) finalize(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) (reconcile.Result, error) { + stored := nodeClass.DeepCopy() + if !controllerutil.ContainsFinalizer(nodeClass, v1alpha2.TerminationFinalizer) { + return reconcile.Result{}, nil + } + nodeClaimList := &karpv1.NodeClaimList{} + if err := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { + return reconcile.Result{}, fmt.Errorf("listing nodeclaims that are using nodeclass, %w", err) + } + if len(nodeClaimList.Items) > 0 { + c.recorder.Publish(WaitingOnNodeClaimTerminationEvent(nodeClass, lo.Map(nodeClaimList.Items, func(nc karpv1.NodeClaim, _ int) string { return nc.Name }))) + return reconcile.Result{RequeueAfter: time.Minute * 10}, nil // periodically fire the event + } + + // any other processing before removing NodeClass goes here + + controllerutil.RemoveFinalizer(nodeClass, v1alpha2.TerminationFinalizer) + if !equality.Semantic.DeepEqual(stored, nodeClass) { + // We use client.MergeFromWithOptimisticLock because patching a list with a JSON merge patch + // can cause races due to the fact that it fully replaces the list on a change + // Here, we are updating the finalizer list + // https://github.com/kubernetes/kubernetes/issues/111643#issuecomment-2016489732 + if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFromWithOptions(stored, client.MergeFromWithOptimisticLock{})); err != nil { + if errors.IsConflict(err) { + return reconcile.Result{Requeue: true}, nil + } + return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("removing termination finalizer, %w", err)) + } + } + return reconcile.Result{}, nil +} + +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.termination"). + For(&v1alpha2.AKSNodeClass{}). + Watches( + &karpv1.NodeClaim{}, + handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { + nc := o.(*karpv1.NodeClaim) + if nc.Spec.NodeClassRef == nil { + return nil + } + return []reconcile.Request{{NamespacedName: types.NamespacedName{Name: nc.Spec.NodeClassRef.Name}}} + }), + // Watch for NodeClaim deletion events + builder.WithPredicates(predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return false }, + UpdateFunc: func(e event.UpdateEvent) bool { return false }, + DeleteFunc: func(e event.DeleteEvent) bool { return true }, + }), + ). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) +} diff --git a/pkg/controllers/nodeclass/termination/events.go b/pkg/controllers/nodeclass/termination/events.go new file mode 100644 index 000000000..f255e16e6 --- /dev/null +++ b/pkg/controllers/nodeclass/termination/events.go @@ -0,0 +1,38 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package termination + +import ( + "fmt" + + corev1 "k8s.io/api/core/v1" + + "sigs.k8s.io/karpenter/pkg/events" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/utils" +) + +func WaitingOnNodeClaimTerminationEvent(nodeClass *v1alpha2.AKSNodeClass, names []string) events.Event { + return events.Event{ + InvolvedObject: nodeClass, + Type: corev1.EventTypeNormal, + Reason: "WaitingOnNodeClaimTermination", + Message: fmt.Sprintf("Waiting on NodeClaim termination for %s", utils.PrettySlice(names, 5)), + DedupeValues: []string{string(nodeClass.UID)}, + } +} diff --git a/pkg/controllers/nodeclass/termination/suite_test.go b/pkg/controllers/nodeclass/termination/suite_test.go new file mode 100644 index 000000000..2e2e59f70 --- /dev/null +++ b/pkg/controllers/nodeclass/termination/suite_test.go @@ -0,0 +1,124 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package termination_test + +import ( + "context" + "testing" + "time" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + //used for launch template tests until they are migrated + + "github.com/awslabs/operatorpkg/object" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/events" + coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/Azure/karpenter-provider-azure/pkg/apis" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/termination" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" + "github.com/Azure/karpenter-provider-azure/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" +) + +var ctx context.Context +var env *coretest.Environment +var awsEnv *test.Environment +var terminationController *termination.Controller + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "AKSNodeClass") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.AKSNodeClassFieldIndexer(ctx))) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) + awsEnv = test.NewEnvironment(ctx, env) + + terminationController = termination.NewController(env.Client, events.NewRecorder(&record.FakeRecorder{})) +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + awsEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = Describe("NodeClass Termination", func() { + var nodeClass *v1alpha2.AKSNodeClass + BeforeEach(func() { + nodeClass = test.AKSNodeClass() + }) + + It("should not delete the AKSNodeClass until all associated NodeClaims are terminated", func() { + var nodeClaims []*karpv1.NodeClaim + for i := 0; i < 2; i++ { + nc := coretest.NodeClaim(karpv1.NodeClaim{ + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nc) + nodeClaims = append(nodeClaims, nc) + } + controllerutil.AddFinalizer(nodeClass, v1alpha2.TerminationFinalizer) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) + + Expect(env.Client.Delete(ctx, nodeClass)).To(Succeed()) + res := ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) + Expect(res.RequeueAfter).To(Equal(time.Minute * 10)) + ExpectExists(ctx, env.Client, nodeClass) + + // Delete one of the NodeClaims + // The NodeClass should still not delete + ExpectDeleted(ctx, env.Client, nodeClaims[0]) + res = ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) + Expect(res.RequeueAfter).To(Equal(time.Minute * 10)) + ExpectExists(ctx, env.Client, nodeClass) + + // Delete the last NodeClaim + // The NodeClass should now delete + ExpectDeleted(ctx, env.Client, nodeClaims[1]) + ExpectObjectReconciled(ctx, env.Client, terminationController, nodeClass) + ExpectNotFound(ctx, env.Client, nodeClass) + }) +}) diff --git a/pkg/test/aksnodeclass.go b/pkg/test/aksnodeclass.go index 374625cdc..32ce145e0 100644 --- a/pkg/test/aksnodeclass.go +++ b/pkg/test/aksnodeclass.go @@ -17,11 +17,15 @@ limitations under the License. package test import ( + "context" "fmt" "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/imdario/mergo" "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) @@ -44,3 +48,15 @@ func AKSNodeClass(overrides ...v1alpha2.AKSNodeClass) *v1alpha2.AKSNodeClass { nc.Spec.ImageFamily = lo.ToPtr(v1alpha2.Ubuntu2204ImageFamily) return nc } + +func AKSNodeClassFieldIndexer(ctx context.Context) func(cache.Cache) error { + return func(c cache.Cache) error { + return c.IndexField(ctx, &karpv1.NodeClaim{}, "spec.nodeClassRef.name", func(obj client.Object) []string { + nc := obj.(*karpv1.NodeClaim) + if nc.Spec.NodeClassRef == nil { + return []string{""} + } + return []string{nc.Spec.NodeClassRef.Name} + }) + } +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 58eecc595..f1a1ff692 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -22,6 +22,7 @@ import ( "os" "regexp" "strconv" + "strings" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute" v1 "k8s.io/api/core/v1" @@ -121,3 +122,19 @@ func StringMap(list v1.ResourceList) map[string]string { } return m } + +// PrettySlice truncates a slice after a certain number of max items to ensure +// that the Slice isn't too long +func PrettySlice[T any](s []T, maxItems int) string { + var sb strings.Builder + for i, elem := range s { + if i > maxItems-1 { + fmt.Fprintf(&sb, " and %d other(s)", len(s)-i) + break + } else if i > 0 { + fmt.Fprint(&sb, ", ") + } + fmt.Fprint(&sb, elem) + } + return sb.String() +} From 9e6f8e27e3159cb260950c4624177386def1317c Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 11 Oct 2024 23:07:32 +0000 Subject: [PATCH 25/47] fix(lint): restore linting on verify --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 24a23ea2c..4106f49a6 100644 --- a/Makefile +++ b/Makefile @@ -87,8 +87,7 @@ verify: toolchain tidy download ## Verify code. Includes dependencies, linting, cp pkg/apis/crds/* charts/karpenter-crd/templates hack/mutation/conversion_webhooks_injection.sh hack/github/dependabot.sh - # TODO: restore linting, excluding code generators (typecheck "main redeclared" issue) - # $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) + $(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline)) @git diff --quiet ||\ { echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print " - " $$0}'; \ if [ "${CI}" = true ]; then\ From 2f15c9fe6f0060a8705250f564bf5e7267ef23b6 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:16:31 +0000 Subject: [PATCH 26/47] feat: add nodeclass hash controller --- pkg/apis/v1alpha2/aksnodeclass.go | 7 + pkg/apis/v1alpha2/labels.go | 3 + pkg/controllers/nodeclass/hash/controller.go | 120 +++++++++ pkg/controllers/nodeclass/hash/suite_test.go | 254 +++++++++++++++++++ 4 files changed, 384 insertions(+) create mode 100644 pkg/controllers/nodeclass/hash/controller.go create mode 100644 pkg/controllers/nodeclass/hash/suite_test.go diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index 916bbf264..968d63632 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -137,6 +137,13 @@ type AKSNodeClass struct { Status AKSNodeClassStatus `json:"status,omitempty"` } +// We need to bump the AKSNodeClassHashVersion when we make an update to the AKSNodeClass CRD under these conditions: +// 1. A field changes its default value for an existing field that is already hashed +// 2. A field is added to the hash calculation with an already-set value +// 3. A field is removed from the hash calculations +const AKSNodeClassHashVersion = "v1" + +// TODO: add hash tests func (in *AKSNodeClass) Hash() string { return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ SlicesAsSets: true, diff --git a/pkg/apis/v1alpha2/labels.go b/pkg/apis/v1alpha2/labels.go index b80cbf023..fd91fb202 100644 --- a/pkg/apis/v1alpha2/labels.go +++ b/pkg/apis/v1alpha2/labels.go @@ -104,6 +104,9 @@ var ( AKSLabelDomain = "kubernetes.azure.com" AKSLabelCluster = AKSLabelDomain + "/cluster" + + AnnotationAKSNodeClassHash = apis.Group + "/aksnodeclass-hash" + AnnotationAKSNodeClassHashVersion = apis.Group + "/aksnodeclass-hash-version" ) const ( diff --git a/pkg/controllers/nodeclass/hash/controller.go b/pkg/controllers/nodeclass/hash/controller.go new file mode 100644 index 000000000..8948e0a29 --- /dev/null +++ b/pkg/controllers/nodeclass/hash/controller.go @@ -0,0 +1,120 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hash + +import ( + "context" + + "github.com/samber/lo" + "go.uber.org/multierr" + "k8s.io/apimachinery/pkg/api/equality" + controllerruntime "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/operator/injection" + + "github.com/awslabs/operatorpkg/reasonable" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" +) + +type Controller struct { + kubeClient client.Client +} + +func NewController(kubeClient client.Client) *Controller { + return &Controller{ + kubeClient: kubeClient, + } +} + +func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) (reconcile.Result, error) { + ctx = injection.WithControllerName(ctx, "nodeclass.hash") + + stored := nodeClass.DeepCopy() + + if nodeClass.Annotations[v1alpha2.AnnotationAKSNodeClassHashVersion] != v1alpha2.AKSNodeClassHashVersion { + if err := c.updateNodeClaimHash(ctx, nodeClass); err != nil { + return reconcile.Result{}, err + } + } + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: nodeClass.Hash(), + v1alpha2.AnnotationAKSNodeClassHashVersion: v1alpha2.AKSNodeClassHashVersion, + }) + + if !equality.Semantic.DeepEqual(stored, nodeClass) { + if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { + return reconcile.Result{}, err + } + } + + return reconcile.Result{}, nil +} + +func (c *Controller) Register(_ context.Context, m manager.Manager) error { + return controllerruntime.NewControllerManagedBy(m). + Named("nodeclass.hash"). + For(&v1alpha2.AKSNodeClass{}). + WithOptions(controller.Options{ + RateLimiter: reasonable.RateLimiter(), + MaxConcurrentReconciles: 10, + }). + Complete(reconcile.AsReconciler(m.GetClient(), c)) +} + +// Updating `AKSNodeClass-hash-version` annotation inside the karpenter controller means a breaking change has been made to the hash calculation. +// `AKSNodeClass-hash` annotation on the AKSNodeClass will be updated, due to the breaking change, making the `AKSNodeClass-hash` on the NodeClaim different from +// AKSNodeClass. Since, we cannot rely on the `AKSNodeClass-hash` on the NodeClaims, due to the breaking change, we will need to re-calculate the hash and update the annotation. +// For more information on the Drift Hash Versioning: https://github.com/kubernetes-sigs/karpenter/blob/main/designs/drift-hash-versioning.md +func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass) error { + ncList := &karpv1.NodeClaimList{} + if err := c.kubeClient.List(ctx, ncList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { + return err + } + + errs := make([]error, len(ncList.Items)) + for i := range ncList.Items { + nc := ncList.Items[i] + stored := nc.DeepCopy() + + if nc.Annotations[v1alpha2.AnnotationAKSNodeClassHashVersion] != v1alpha2.AKSNodeClassHashVersion { + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1alpha2.AnnotationAKSNodeClassHashVersion: v1alpha2.AKSNodeClassHashVersion, + }) + + // Any NodeClaim that is already drifted will remain drifted if the karpenter.k8s.aws/nodepool-hash-version doesn't match + // Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed + if nc.StatusConditions().Get(karpv1.ConditionTypeDrifted) == nil { + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: nodeClass.Hash(), + }) + } + + if !equality.Semantic.DeepEqual(stored, nc) { + if err := c.kubeClient.Patch(ctx, &nc, client.MergeFrom(stored)); err != nil { + errs[i] = client.IgnoreNotFound(err) + } + } + } + } + + return multierr.Combine(errs...) +} diff --git a/pkg/controllers/nodeclass/hash/suite_test.go b/pkg/controllers/nodeclass/hash/suite_test.go new file mode 100644 index 000000000..0fac30d3e --- /dev/null +++ b/pkg/controllers/nodeclass/hash/suite_test.go @@ -0,0 +1,254 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hash_test + +import ( + "context" + "testing" + + "sigs.k8s.io/karpenter/pkg/test/v1alpha1" + + "github.com/awslabs/operatorpkg/object" + "github.com/imdario/mergo" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" + coreoptions "sigs.k8s.io/karpenter/pkg/operator/options" + coretest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/Azure/karpenter-provider-azure/pkg/apis" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" + "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/hash" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" + "github.com/Azure/karpenter-provider-azure/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "sigs.k8s.io/karpenter/pkg/test/expectations" + . "sigs.k8s.io/karpenter/pkg/utils/testing" +) + +var ctx context.Context +var env *coretest.Environment +var awsEnv *test.Environment +var hashController *hash.Controller + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "AKSNodeClass") +} + +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(coretest.WithCRDs(apis.CRDs...), coretest.WithCRDs(v1alpha1.CRDs...), coretest.WithFieldIndexers(test.AKSNodeClassFieldIndexer(ctx))) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) + awsEnv = test.NewEnvironment(ctx, env) + + hashController = hash.NewController(env.Client) +}) + +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") +}) + +var _ = BeforeEach(func() { + ctx = coreoptions.ToContext(ctx, coretest.Options()) + awsEnv.Reset() +}) + +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) + +var _ = Describe("NodeClass Hash Controller", func() { + var nodeClass *v1alpha2.AKSNodeClass + var nodePool *karpv1.NodePool + BeforeEach(func() { + nodeClass = test.AKSNodeClass() + nodePool = coretest.NodePool(karpv1.NodePool{ + Spec: karpv1.NodePoolSpec{ + Template: karpv1.NodeClaimTemplate{ + Spec: karpv1.NodeClaimTemplateSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }, + }, + }) + }) + DescribeTable("should update the drift hash when static field is updated", func(changes *v1alpha2.AKSNodeClass) { + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + expectedHash := nodeClass.Hash() + Expect(nodeClass.ObjectMeta.Annotations[v1alpha2.AnnotationAKSNodeClassHash]).To(Equal(expectedHash)) + + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + + ExpectApplied(ctx, env.Client, nodeClass) + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + expectedHashTwo := nodeClass.Hash() + Expect(nodeClass.Annotations[v1alpha2.AnnotationAKSNodeClassHash]).To(Equal(expectedHashTwo)) + Expect(expectedHash).ToNot(Equal(expectedHashTwo)) + + }, + Entry("ImageFamily Drift", &v1alpha2.AKSNodeClass{Spec: v1alpha2.AKSNodeClassSpec{ImageFamily: lo.ToPtr("AzureLinux")}}), + Entry("OSDiskSizeGB Drift", &v1alpha2.AKSNodeClass{Spec: v1alpha2.AKSNodeClassSpec{OSDiskSizeGB: lo.ToPtr(int32(100))}}), + Entry("Tags Drift", &v1alpha2.AKSNodeClass{Spec: v1alpha2.AKSNodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + ) + It("should update AKSNodeClass-hash-version annotation when the AKSNodeClass-hash-version on the NodeClass does not match with the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "abceduefed", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + } + ExpectApplied(ctx, env.Client, nodeClass) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + expectedHash := nodeClass.Hash() + // Expect AKSNodeClass-hash on the NodeClass to be updated + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + }) + It("should update AKSNodeClass-hash-versions on all NodeClaims when the AKSNodeClass-hash-version does not match with the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "abceduefed", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + } + nodeClaimOne := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "123456", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + nodeClaimTwo := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "123456", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + + ExpectApplied(ctx, env.Client, nodeClass, nodeClaimOne, nodeClaimTwo, nodePool) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + nodeClaimOne = ExpectExists(ctx, env.Client, nodeClaimOne) + nodeClaimTwo = ExpectExists(ctx, env.Client, nodeClaimTwo) + + expectedHash := nodeClass.Hash() + // Expect AKSNodeClass-hash on the NodeClaims to be updated + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, expectedHash)) + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, expectedHash)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + }) + It("should not update AKSNodeClass-hash on all NodeClaims when the AKSNodeClass-hash-version matches the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "abceduefed", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test-version", + } + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "1234564654", + v1alpha2.AnnotationAKSNodeClassHashVersion: v1alpha2.AKSNodeClassHashVersion, + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + + expectedHash := nodeClass.Hash() + + // Expect AKSNodeClass-hash on the NodeClass to be updated + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + // Expect AKSNodeClass-hash on the NodeClaims to stay the same + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, "1234564654")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + }) + It("should not update AKSNodeClass-hash on the NodeClaim if it's drifted and the AKSNodeClass-hash-version does not match the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "abceduefed", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + } + nodeClaim := coretest.NodeClaim(karpv1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{karpv1.NodePoolLabelKey: nodePool.Name}, + Annotations: map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: "123456", + v1alpha2.AnnotationAKSNodeClassHashVersion: "test", + }, + }, + Spec: karpv1.NodeClaimSpec{ + NodeClassRef: &karpv1.NodeClassReference{ + Group: object.GVK(nodeClass).Group, + Kind: object.GVK(nodeClass).Kind, + Name: nodeClass.Name, + }, + }, + }) + nodeClaim.StatusConditions().SetTrue(karpv1.ConditionTypeDrifted) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim, nodePool) + + ExpectObjectReconciled(ctx, env.Client, hashController, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + + // Expect AKSNodeClass-hash on the NodeClaims to stay the same + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHash, "123456")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1alpha2.AnnotationAKSNodeClassHashVersion, v1alpha2.AKSNodeClassHashVersion)) + }) +}) From 8ce048232ecf5d6b0885102e4cf1acc0f5d63dcc Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 00:20:23 +0000 Subject: [PATCH 27/47] fix: register additional nodeclass and status controllers --- cmd/controller/main.go | 2 ++ cmd/controller/main_ccp.go | 2 ++ pkg/controllers/controllers.go | 16 ++++++++++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 3977d695f..24d9b5207 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -56,7 +56,9 @@ func main() { WithWebhooks(ctx, corewebhooks.NewWebhooks()...). WithControllers(ctx, controllers.NewControllers( ctx, + op.Manager, op.GetClient(), + op.EventRecorder, aksCloudProvider, op.InstanceProvider, )...). diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 2005ac851..91401209c 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -59,7 +59,9 @@ func main() { // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). WithControllers(ctx, controllers.NewControllers( ctx, + op.Manager, op.GetClient(), + op.EventRecorder, aksCloudProvider, op.InstanceProvider, )...). diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 2758c9f51..e96be8104 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -20,22 +20,34 @@ import ( "context" "github.com/awslabs/operatorpkg/controller" + "github.com/awslabs/operatorpkg/status" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/manager" + "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" nodeclaimgarbagecollection "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/garbagecollection" + nodeclasshash "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/hash" nodeclassstatus "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/status" + nodeclasstermination "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclass/termination" "github.com/Azure/karpenter-provider-azure/pkg/controllers/nodeclaim/inplaceupdate" "github.com/Azure/karpenter-provider-azure/pkg/providers/instance" ) -func NewControllers(ctx context.Context, kubeClient client.Client, cloudProvider cloudprovider.CloudProvider, instanceProvider instance.Provider) []controller.Controller { +func NewControllers(ctx context.Context, mgr manager.Manager, kubeClient client.Client, recorder events.Recorder, + cloudProvider cloudprovider.CloudProvider, instanceProvider instance.Provider) []controller.Controller { controllers := []controller.Controller{ + nodeclasshash.NewController(kubeClient), + nodeclassstatus.NewController(kubeClient), + nodeclasstermination.NewController(kubeClient, recorder), nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), + // TODO: nodeclaim tagging inplaceupdate.NewController(kubeClient, instanceProvider), - nodeclassstatus.NewController(kubeClient), + status.NewController[*v1alpha2.AKSNodeClass](kubeClient, mgr.GetEventRecorderFor("karpenter")), } return controllers } From d4fac7f48102bbcaf2c38c4edf96ccc4e7c282f2 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 04:43:41 +0000 Subject: [PATCH 28/47] fix(e2e): better selection of karpenter pod for logs --- .github/actions/e2e/dump-logs/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml index 85c95db1f..8914281ad 100644 --- a/.github/actions/e2e/dump-logs/action.yaml +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -39,7 +39,7 @@ runs: run: | echo "step: controller-logs" AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - POD_NAME=$(kubectl get pods -n kube-system --no-headers -o custom-columns=":metadata.name" | tail -n 1) + POD_NAME=$(kubectl get pods -n kube-system -l app.kubernetes.io/name=karpenter -o jsonpath='{.items[*].metadata.name}') echo "logs from pod ${POD_NAME}" kubectl logs "${POD_NAME}" -n kube-system -c controller - name: describe-karpenter-pods @@ -47,7 +47,7 @@ runs: run: | echo "step: describe-karpenter-pods" AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - kubectl describe pods -n kube-system + kubectl describe pods -n kube-system -l app.kubernetes.io/name=karpenter - name: describe-nodes shell: bash run: | From 1f4da31f9c2d1fc6718a73201fd12b58e71f1e65 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 05:44:10 +0000 Subject: [PATCH 29/47] fix(e2e): fix utilization suite --- test/suites/utilization/suite_test.go | 70 ++++++++++++++------------- 1 file changed, 37 insertions(+), 33 deletions(-) diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go index b114c3f8d..dc50b4357 100644 --- a/test/suites/utilization/suite_test.go +++ b/test/suites/utilization/suite_test.go @@ -47,40 +47,44 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Utilization", func() { - azLinuxNodeClass := env.AZLinuxNodeClass() - ubuntuNodeClass := env.DefaultAKSNodeClass() - azLinuxNodeClassArm := env.AZLinuxNodeClass() - ubuntuNodeClassArm := env.DefaultAKSNodeClass() - - DescribeTable("should provision one pod per node", - func(nodeClass *v1alpha2.AKSNodeClass, nodePool *karpv1.NodePool) { - test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ - NodeSelectorRequirement: v1.NodeSelectorRequirement{ - Key: v1alpha2.LabelSKUCPU, - Operator: v1.NodeSelectorOpLt, - Values: []string{"3"}, - }}) + It("should provision one pod per node (AzureLinux, amd64)", func() { + ExpectProvisionPodPerNode(env.AZLinuxNodeClass, env.DefaultNodePool) + }) + It("should provision one pod per node (AzureLinux, arm64)", func() { + ExpectProvisionPodPerNode(env.AZLinuxNodeClass, env.ArmNodepool) + }) + It("should provision one pod per node (Ubuntu, amd64)", func() { + ExpectProvisionPodPerNode(env.DefaultAKSNodeClass, env.DefaultNodePool) + }) + It("should provision one pod per node (Ubuntu, arm64)", func() { + ExpectProvisionPodPerNode(env.DefaultAKSNodeClass, env.ArmNodepool) + }) +}) - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 10, - PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1.1"), - }, - }, - Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", +func ExpectProvisionPodPerNode(getNodeClass func() *v1alpha2.AKSNodeClass, getNodePool func(*v1alpha2.AKSNodeClass) *karpv1.NodePool) { + GinkgoHelper() + nodeClass := getNodeClass() + nodePool := getNodePool(nodeClass) + test.ReplaceRequirements(nodePool, karpv1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1alpha2.LabelSKUCPU, + Operator: v1.NodeSelectorOpLt, + Values: []string{"3"}, + }}) + + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 10, + PodOptions: test.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1.1"), }, - }) - - env.ExpectCreated(nodePool, nodeClass, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) // One pod per node enforced by instance size + }, + Image: "mcr.microsoft.com/oss/kubernetes/pause:3.6", }, + }) - Entry("should provision one pod per node (AzureLinux, amd64)", azLinuxNodeClass, env.DefaultNodePool(azLinuxNodeClass)), - Entry("should provision one pod per node (AzureLinux, arm64)", azLinuxNodeClassArm, env.ArmNodepool(azLinuxNodeClassArm)), - Entry("should provision one pod per node (Ubuntu, amd64)", ubuntuNodeClass, env.DefaultNodePool(ubuntuNodeClass)), - Entry("should provision one pod per node (Ubuntu, arm64)", ubuntuNodeClassArm, env.ArmNodepool(ubuntuNodeClassArm)), - ) -}) + env.ExpectCreated(nodePool, nodeClass, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", int(*deployment.Spec.Replicas)) // One pod per node enforced by instance size +} From 1ac70354cbbc89508fc7a35d78f71de9fb705afd Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 05:45:40 +0000 Subject: [PATCH 30/47] chore(e2e): add events to dump-logs (and simplify) --- .github/actions/e2e/dump-logs/action.yaml | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml index 8914281ad..2d6e49812 100644 --- a/.github/actions/e2e/dump-logs/action.yaml +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -31,26 +31,23 @@ runs: client-id: ${{ inputs.client-id }} tenant-id: ${{ inputs.tenant-id }} subscription-id: ${{ inputs.subscription-id }} - - name: az set sub + - name: update cluster context shell: bash - run: az account set --subscription ${{ inputs.subscription-id }} + run: | + az aks get-credentials --name ${{ inputs.cluster_name }} --resource-group $${{ inputs.resource_group }} - name: controller-logs shell: bash run: | - echo "step: controller-logs" - AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds - POD_NAME=$(kubectl get pods -n kube-system -l app.kubernetes.io/name=karpenter -o jsonpath='{.items[*].metadata.name}') - echo "logs from pod ${POD_NAME}" - kubectl logs "${POD_NAME}" -n kube-system -c controller + kubectl logs -n kube-system -l app.kubernetes.io/name=karpenter --all-containers --ignore-errors - name: describe-karpenter-pods shell: bash run: | - echo "step: describe-karpenter-pods" - AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds kubectl describe pods -n kube-system -l app.kubernetes.io/name=karpenter - name: describe-nodes shell: bash run: | - echo "step: describe-nodes" - AZURE_CLUSTER_NAME=${{ inputs.cluster_name }} AZURE_RESOURCE_GROUP=${{ inputs.resource_group }} make az-creds kubectl describe nodes + - name: get-karpenter-events + shell: bash + run: | + kubectl get events -A --field-selector source=karpenter From 6d9edf064d53bbeba0809168528a10e44466c73f Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 05:59:01 +0000 Subject: [PATCH 31/47] chore: rename v1 to corev1 --- test/suites/integration/daemonset_test.go | 36 +++++++++++------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index d4368c5b6..9bd4bd80b 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -18,7 +18,7 @@ package integration_test import ( appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" schedulingv1 "k8s.io/api/scheduling/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -32,7 +32,7 @@ import ( ) var _ = Describe("DaemonSet", func() { - var limitrange *v1.LimitRange + var limitrange *corev1.LimitRange var priorityclass *schedulingv1.PriorityClass var daemonset *appsv1.DaemonSet var dep *appsv1.Deployment @@ -48,7 +48,7 @@ var _ = Describe("DaemonSet", func() { GlobalDefault: false, Description: "This priority class should be used for daemonsets.", } - limitrange = &v1.LimitRange{ + limitrange = &corev1.LimitRange{ ObjectMeta: metav1.ObjectMeta{ Name: "limitrange", Namespace: "default", @@ -56,7 +56,7 @@ var _ = Describe("DaemonSet", func() { } daemonset = test.DaemonSet(test.DaemonSetOptions{ PodOptions: test.PodOptions{ - ResourceRequirements: v1.ResourceRequirements{Limits: v1.ResourceList{v1.ResourceMemory: resource.MustParse("1Gi")}}, + ResourceRequirements: corev1.ResourceRequirements{Limits: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("1Gi")}}, PriorityClassName: "high-priority-daemonsets", }, }) @@ -67,19 +67,19 @@ var _ = Describe("DaemonSet", func() { ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"app": "large-app"}, }, - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{v1.ResourceMemory: resource.MustParse("4")}, + ResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceMemory: resource.MustParse("4")}, }, }, }) }) It("should account for LimitRange Default on daemonSet pods for resources", func() { - limitrange.Spec.Limits = []v1.LimitRangeItem{ + limitrange.Spec.Limits = []corev1.LimitRangeItem{ { - Type: v1.LimitTypeContainer, - Default: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("1Gi"), + Type: corev1.LimitTypeContainer, + Default: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, } @@ -90,7 +90,7 @@ var _ = Describe("DaemonSet", func() { // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it Eventually(func(g Gomega) { - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) g.Expect(nodeList.Items).To(HaveLen(1)) @@ -105,12 +105,12 @@ var _ = Describe("DaemonSet", func() { }).Should(Succeed()) }) It("should account for LimitRange DefaultRequest on daemonSet pods for resources", func() { - limitrange.Spec.Limits = []v1.LimitRangeItem{ + limitrange.Spec.Limits = []corev1.LimitRangeItem{ { - Type: v1.LimitTypeContainer, - DefaultRequest: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("1Gi"), + Type: corev1.LimitTypeContainer, + DefaultRequest: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("1Gi"), }, }, } @@ -121,7 +121,7 @@ var _ = Describe("DaemonSet", func() { // Eventually expect a single node to exist and both the deployment pod and the daemonset pod to schedule to it Eventually(func(g Gomega) { - nodeList := &v1.NodeList{} + nodeList := &corev1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{"testing/cluster"})).To(Succeed()) g.Expect(nodeList.Items).To(HaveLen(1)) From a06dcc944f3786741e4db0fcdc47b7af18214b1a Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Sat, 12 Oct 2024 06:24:44 +0000 Subject: [PATCH 32/47] fix: remove extra $ --- .github/actions/e2e/dump-logs/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml index 2d6e49812..570c8cba4 100644 --- a/.github/actions/e2e/dump-logs/action.yaml +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -34,7 +34,7 @@ runs: - name: update cluster context shell: bash run: | - az aks get-credentials --name ${{ inputs.cluster_name }} --resource-group $${{ inputs.resource_group }} + az aks get-credentials --name ${{ inputs.cluster_name }} --resource-group ${{ inputs.resource_group }} - name: controller-logs shell: bash run: | From d7b6df9475185a2405361b7060dacd21ed979e8b Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:50:31 +0000 Subject: [PATCH 33/47] fix(e2e): add cilium label and taint --- test/pkg/environment/common/environment.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index 67cb5fe49..a18a40a2d 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -178,6 +178,18 @@ func (env *Environment) DefaultNodePool(nodeClass *v1alpha2.AKSNodeClass) *karpv corev1.ResourceCPU: resource.MustParse("1000"), // TODO: do we need that much? corev1.ResourceMemory: resource.MustParse("1000Gi"), }) + + // TODO: make this conditional on Cilium + // https://karpenter.sh/docs/concepts/nodepools/#cilium-startup-taint + nodePool.Spec.Template.Spec.StartupTaints = append(nodePool.Spec.Template.Spec.StartupTaints, corev1.Taint{ + Key: "node.cilium.io/agent-not-ready", + Effect: corev1.TaintEffectNoExecute, + Value: "true", + }) + // # required for Karpenter to predict overhead from cilium DaemonSet + nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{ + "kubernetes.azure.com/ebpf-dataplane": "cilium", + }) return nodePool } From 326817db152fc30f06ecf5a1df3aec6c6c9e6e60 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Tue, 15 Oct 2024 23:51:31 +0000 Subject: [PATCH 34/47] fix(e2e): fix labels and disruption for deamonset test --- test/suites/integration/daemonset_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index 9bd4bd80b..85479f942 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -26,8 +26,10 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" "sigs.k8s.io/controller-runtime/pkg/client" + karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" "sigs.k8s.io/karpenter/pkg/test" ) @@ -38,7 +40,9 @@ var _ = Describe("DaemonSet", func() { var dep *appsv1.Deployment BeforeEach(func() { - nodePool.Spec.Template.Labels = map[string]string{"testing/cluster": "test"} + nodePool.Spec.Disruption.ConsolidationPolicy = karpv1.ConsolidationPolicyWhenEmptyOrUnderutilized + nodePool.Spec.Disruption.ConsolidateAfter = karpv1.MustParseNillableDuration("0s") + nodePool.Spec.Template.Labels = lo.Assign(nodePool.Spec.Template.Labels, map[string]string{"testing/cluster": "test"}) priorityclass = &schedulingv1.PriorityClass{ ObjectMeta: metav1.ObjectMeta{ From 9429725334f93647bcd55fc548bbed4b1b74d6c6 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:32:28 +0000 Subject: [PATCH 35/47] feat: update kubelet configuration --- .../karpenter.azure.com_aksnodeclasses.yaml | 460 +++++++++--------- hack/validation/kubelet.sh | 14 +- .../karpenter.azure.com_aksnodeclasses.yaml | 460 +++++++++--------- pkg/apis/v1alpha2/aksnodeclass.go | 108 ++-- pkg/apis/v1alpha2/zz_generated.deepcopy.go | 86 ++-- pkg/providers/imagefamily/azlinux.go | 2 +- .../imagefamily/bootstrap/aksbootstrap.go | 10 +- .../imagefamily/bootstrap/bootstrap.go | 23 +- pkg/providers/imagefamily/resolver.go | 21 +- pkg/providers/imagefamily/ubuntu_2204.go | 2 +- pkg/providers/instancetype/instancetype.go | 15 +- pkg/providers/instancetype/suite_test.go | 78 +-- 12 files changed, 621 insertions(+), 658 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index f47e51108..05c69e24d 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -9,247 +9,251 @@ spec: group: karpenter.azure.com names: categories: - - karpenter + - karpenter kind: AKSNodeClass listKind: AKSNodeClassList plural: aksnodeclasses shortNames: - - aksnc - - aksncs + - aksnc + - aksncs singular: aksnodeclass scope: Cluster versions: - - name: v1alpha2 - schema: - openAPIV3Schema: - description: AKSNodeClass is the Schema for the AKSNodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. - This will contain configuration necessary to launch instances in AKS. - properties: - imageFamily: - default: Ubuntu2204 - description: ImageFamily is the image family that instances use. - enum: - - Ubuntu2204 - - AzureLinux - type: string - imageVersion: - description: ImageVersion is the image version that instances use. + - name: v1alpha2 + schema: + openAPIV3Schema: + description: AKSNodeClass is the Schema for the AKSNodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. + This will contain configuration necessary to launch instances in AKS. + properties: + imageFamily: + default: Ubuntu2204 + description: ImageFamily is the image family that instances use. + enum: + - Ubuntu2204 + - AzureLinux + type: string + imageVersion: + description: ImageVersion is the image version that instances use. + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + allowedUnsafeSysctls: + description: |- + A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `*`). + Unsafe sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, + and `net.*`. For example: "`kernel.msg*,net.ipv4.route.min_pmtu`" + Default: [] + items: + type: string + type: array + containerLogMaxFiles: + default: 5 + description: |- + containerLogMaxFiles specifies the maximum number of container log files that can be present for a container. + Default: 5 + format: int32 + minimum: 2 + type: integer + containerLogMaxSize: + default: 50Mi + description: |- + containerLogMaxSize is a quantity defining the maximum size of the container log + file before it is rotated. For example: "5Mi" or "256Ki". + Default: "10Mi" + AKS CustomKubeletConfig has containerLogMaxSizeMB (with units), defaults to 50 + pattern: ^\d+(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$ + type: string + cpuCFSQuota: + default: true + description: |- + CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + Note: AKS CustomKubeletConfig uses cpuCfsQuota (camelCase) + type: boolean + cpuCFSQuotaPeriod: + default: 100ms + description: |- + cpuCfsQuotaPeriod sets the CPU CFS quota period value, `cpu.cfs_period_us`. + The value must be between 1 ms and 1 second, inclusive. + Default: "100ms" + type: string + cpuManagerPolicy: + default: none + description: cpuManagerPolicy is the name of the policy to use. + enum: + - none + - static + type: string + failSwapOn: + description: |- + failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. + Default: true + kubebuilder:default:=true + type: boolean + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + Note: AKS CustomKubeletConfig does not have "Percent" in the field name + format: int32 + maximum: 100 + minimum: 0 + type: integer + podPidsLimit: + description: |- + podPidsLimit is the maximum number of PIDs in any pod. + AKS CustomKubeletConfig uses PodMaxPids, int32 (!) + Default: -1 + format: int64 + type: integer + topologyManagerPolicy: + default: none + description: |- + topologyManagerPolicy is the name of the topology manager policy to use. + Valid values include: + + - `restricted`: kubelet only allows pods with optimal NUMA node alignment for requested resources; + - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device resources; + - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + of CPU and device resources. + enum: + - restricted + - best-effort + - none + - single-numa-node + type: string + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + maxPods: + description: MaxPods is an override for the maximum number of pods + that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + osDiskSizeGB: + default: 128 + description: osDiskSizeGB is the size of the OS disk in GB. + format: int32 + minimum: 100 + type: integer + tags: + additionalProperties: type: string - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. + description: Tags to be applied on Azure resources like instances. + type: object + vnetSubnetID: + description: |- + VNETSubnetID is the subnet used by nics provisioned with this nodeclass. + If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config + pattern: (?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$ + type: string + type: object + status: + description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods properties: - clusterDNS: + lastTransitionTime: description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 minimum: 0 type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: + reason: description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - osDiskSizeGB: - default: 128 - description: osDiskSizeGB is the size of the OS disk in GB. - format: int32 - minimum: 100 - type: integer - tags: - additionalProperties: - type: string - description: Tags to be applied on Azure resources like instances. + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type type: object - vnetSubnetID: - description: |- - VNETSubnetID is the subnet used by nics provisioned with this nodeclass. - If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config - pattern: (?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$ - type: string - type: object - status: - description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass - properties: - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/hack/validation/kubelet.sh b/hack/validation/kubelet.sh index a29b44498..a174e2ca2 100755 --- a/hack/validation/kubelet.sh +++ b/hack/validation/kubelet.sh @@ -1,16 +1,4 @@ #!/bin/bash set -euo pipefail -# Kubelet Validation - -# The regular expression adds validation for kubelet.kubeReserved and kubelet.systemReserved values of the map are resource.Quantity -# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 -# AKSNodeClass Validation: -yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.kubeReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml -yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.systemReserved.additionalProperties.pattern = "^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml - -# The regular expression is a validation for kubelet.evictionHard and kubelet.evictionSoft are percentage or a resource.Quantity -# Quantity: https://github.com/kubernetes/apimachinery/blob/d82afe1e363acae0e8c0953b1bc230d65fdb50e2/pkg/api/resource/quantity.go#L100 -# AKSNodeClass Validation: -yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionHard.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml -yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.kubelet.properties.evictionSoft.additionalProperties.pattern = "^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$"' -i pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +# Kubelet Validation (placeholder) diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index f47e51108..05c69e24d 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -9,247 +9,251 @@ spec: group: karpenter.azure.com names: categories: - - karpenter + - karpenter kind: AKSNodeClass listKind: AKSNodeClassList plural: aksnodeclasses shortNames: - - aksnc - - aksncs + - aksnc + - aksncs singular: aksnodeclass scope: Cluster versions: - - name: v1alpha2 - schema: - openAPIV3Schema: - description: AKSNodeClass is the Schema for the AKSNodeClass API - properties: - apiVersion: - description: |- - APIVersion defines the versioned schema of this representation of an object. - Servers should convert recognized schemas to the latest internal value, and - may reject unrecognized values. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources - type: string - kind: - description: |- - Kind is a string value representing the REST resource this object represents. - Servers may infer this from the endpoint the client submits requests to. - Cannot be updated. - In CamelCase. - More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds - type: string - metadata: - type: object - spec: - description: |- - AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. - This will contain configuration necessary to launch instances in AKS. - properties: - imageFamily: - default: Ubuntu2204 - description: ImageFamily is the image family that instances use. - enum: - - Ubuntu2204 - - AzureLinux - type: string - imageVersion: - description: ImageVersion is the image version that instances use. + - name: v1alpha2 + schema: + openAPIV3Schema: + description: AKSNodeClass is the Schema for the AKSNodeClass API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + AKSNodeClassSpec is the top level specification for the AKS Karpenter Provider. + This will contain configuration necessary to launch instances in AKS. + properties: + imageFamily: + default: Ubuntu2204 + description: ImageFamily is the image family that instances use. + enum: + - Ubuntu2204 + - AzureLinux + type: string + imageVersion: + description: ImageVersion is the image version that instances use. + type: string + kubelet: + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. + properties: + allowedUnsafeSysctls: + description: |- + A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `*`). + Unsafe sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, + and `net.*`. For example: "`kernel.msg*,net.ipv4.route.min_pmtu`" + Default: [] + items: + type: string + type: array + containerLogMaxFiles: + default: 5 + description: |- + containerLogMaxFiles specifies the maximum number of container log files that can be present for a container. + Default: 5 + format: int32 + minimum: 2 + type: integer + containerLogMaxSize: + default: 50Mi + description: |- + containerLogMaxSize is a quantity defining the maximum size of the container log + file before it is rotated. For example: "5Mi" or "256Ki". + Default: "10Mi" + AKS CustomKubeletConfig has containerLogMaxSizeMB (with units), defaults to 50 + pattern: ^\d+(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$ + type: string + cpuCFSQuota: + default: true + description: |- + CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + Note: AKS CustomKubeletConfig uses cpuCfsQuota (camelCase) + type: boolean + cpuCFSQuotaPeriod: + default: 100ms + description: |- + cpuCfsQuotaPeriod sets the CPU CFS quota period value, `cpu.cfs_period_us`. + The value must be between 1 ms and 1 second, inclusive. + Default: "100ms" + type: string + cpuManagerPolicy: + default: none + description: cpuManagerPolicy is the name of the policy to use. + enum: + - none + - static + type: string + failSwapOn: + description: |- + failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. + Default: true + kubebuilder:default:=true + type: boolean + imageGCHighThresholdPercent: + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. + Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent + Note: AKS CustomKubeletConfig does not have "Percent" in the field name + format: int32 + maximum: 100 + minimum: 0 + type: integer + podPidsLimit: + description: |- + podPidsLimit is the maximum number of PIDs in any pod. + AKS CustomKubeletConfig uses PodMaxPids, int32 (!) + Default: -1 + format: int64 + type: integer + topologyManagerPolicy: + default: none + description: |- + topologyManagerPolicy is the name of the topology manager policy to use. + Valid values include: + + - `restricted`: kubelet only allows pods with optimal NUMA node alignment for requested resources; + - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device resources; + - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + of CPU and device resources. + enum: + - restricted + - best-effort + - none + - single-numa-node + type: string + type: object + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) + ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : + true' + maxPods: + description: MaxPods is an override for the maximum number of pods + that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + osDiskSizeGB: + default: 128 + description: osDiskSizeGB is the size of the OS disk in GB. + format: int32 + minimum: 100 + type: integer + tags: + additionalProperties: type: string - kubelet: - description: |- - Kubelet defines args to be used when configuring kubelet on provisioned nodes. - They are a subset of the upstream types, recognizing not all options may be supported. - Wherever possible, the types and names should reflect the upstream kubelet types. + description: Tags to be applied on Azure resources like instances. + type: object + vnetSubnetID: + description: |- + VNETSubnetID is the subnet used by nics provisioned with this nodeclass. + If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config + pattern: (?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$ + type: string + type: object + status: + description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass + properties: + conditions: + description: Conditions contains signals for health and readiness + items: + description: Condition aliases the upstream type and adds additional + helper methods properties: - clusterDNS: + lastTransitionTime: description: |- - clusterDNS is a list of IP addresses for the cluster DNS server. - Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionMaxPodGracePeriod: + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: description: |- - EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ - description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - type: object - x-kubernetes-validations: - - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] - rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) - imageGCHighThresholdPercent: - description: |- - ImageGCHighThresholdPercent is the percent of disk usage after which image - garbage collection is always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 and 100, inclusive. - When specified, the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: description: |- - ImageGCLowThresholdPercent is the percent of disk usage before which image - garbage collection is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value by 100, - so the field value must be between 0 and 100, inclusive. - When specified, the value must be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 minimum: 0 type: integer - kubeReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: KubeReserved contains resources reserved for Kubernetes system components. - type: object - x-kubernetes-validations: - - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: kubeReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - maxPods: + reason: description: |- - MaxPods is an override for the maximum number of pods that can run on - a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: |- - PodsPerCore is an override for the number of pods that can run on a worker node - instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - MaxPods is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - description: SystemReserved contains resources reserved for OS system daemons and kernel memory. - type: object - x-kubernetes-validations: - - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] - rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') - - message: systemReserved value cannot be a negative resource quantity - rule: self.all(x, !self[x].startsWith('-')) - type: object - x-kubernetes-validations: - - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent - rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' - - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod - rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true - - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft - rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true - osDiskSizeGB: - default: 128 - description: osDiskSizeGB is the size of the OS disk in GB. - format: int32 - minimum: 100 - type: integer - tags: - additionalProperties: - type: string - description: Tags to be applied on Azure resources like instances. + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type type: object - vnetSubnetID: - description: |- - VNETSubnetID is the subnet used by nics provisioned with this nodeclass. - If not specified, we will use the default --vnet-subnet-id specified in karpenter's options config - pattern: (?i)^\/subscriptions\/[^\/]+\/resourceGroups\/[a-zA-Z0-9_\-().]{0,89}[a-zA-Z0-9_\-()]\/providers\/Microsoft\.Network\/virtualNetworks\/[^\/]+\/subnets\/[^\/]+$ - type: string - type: object - status: - description: AKSNodeClassStatus contains the resolved state of the AKSNodeClass - properties: - conditions: - description: Conditions contains signals for health and readiness - items: - description: Condition aliases the upstream type and adds additional helper methods - properties: - lastTransitionTime: - description: |- - lastTransitionTime is the last time the condition transitioned from one status to another. - This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: |- - message is a human readable message indicating details about the transition. - This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: |- - observedGeneration represents the .metadata.generation that the condition was set based upon. - For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date - with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: |- - reason contains a programmatic identifier indicating the reason for the condition's last transition. - Producers of specific condition types may define expected values and meanings for this field, - and whether the values are considered a guaranteed API. - The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - type: object - type: object - served: true - storage: true - subresources: - status: {} + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index df5be2ee0..4e85537b5 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -53,10 +53,12 @@ type AKSNodeClassSpec struct { // They are a subset of the upstream types, recognizing not all options may be supported. // Wherever possible, the types and names should reflect the upstream kubelet types. // +kubebuilder:validation:XValidation:message="imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent",rule="has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true" - // +kubebuilder:validation:XValidation:message="evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod",rule="has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true" - // +kubebuilder:validation:XValidation:message="evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft",rule="has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true" // +optional Kubelet *KubeletConfiguration `json:"kubelet,omitempty" hash:"ignore"` + // MaxPods is an override for the maximum number of pods that can run on a worker node instance. + // +kubebuilder:validation:Minimum:=0 + // +optional + MaxPods *int32 `json:"maxPods,omitempty"` } // KubeletConfiguration defines args to be used when configuring kubelet on provisioned nodes. @@ -64,52 +66,32 @@ type AKSNodeClassSpec struct { // Wherever possible, the types and names should reflect the upstream kubelet types. // https://pkg.go.dev/k8s.io/kubelet/config/v1beta1#KubeletConfiguration // https://github.com/kubernetes/kubernetes/blob/9f82d81e55cafdedab619ea25cabf5d42736dacf/cmd/kubelet/app/options/options.go#L53 +// +// AKS CustomKubeletConfig w/o CPUReserved,MemoryReserved,SeccompDefault +// https://learn.microsoft.com/en-us/azure/aks/custom-node-configuration?tabs=linux-node-pools type KubeletConfiguration struct { - // clusterDNS is a list of IP addresses for the cluster DNS server. - // Note that not all providers may use all addresses. - //+optional - ClusterDNS []string `json:"clusterDNS,omitempty"` - // MaxPods is an override for the maximum number of pods that can run on - // a worker node instance. - // +kubebuilder:validation:Minimum:=0 - // +optional - MaxPods *int32 `json:"maxPods,omitempty"` - // PodsPerCore is an override for the number of pods that can run on a worker node - // instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if - // MaxPods is a lower value, that value will be used. - // +kubebuilder:validation:Minimum:=0 - // +optional - PodsPerCore *int32 `json:"podsPerCore,omitempty"` - // SystemReserved contains resources reserved for OS system daemons and kernel memory. - // +kubebuilder:validation:XValidation:message="valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" - // +kubebuilder:validation:XValidation:message="systemReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" - // +optional - SystemReserved map[string]string `json:"systemReserved,omitempty"` - // KubeReserved contains resources reserved for Kubernetes system components. - // +kubebuilder:validation:XValidation:message="valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid']",rule="self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid')" - // +kubebuilder:validation:XValidation:message="kubeReserved value cannot be a negative resource quantity",rule="self.all(x, !self[x].startsWith('-'))" - // +optional - KubeReserved map[string]string `json:"kubeReserved,omitempty"` - // EvictionHard is the map of signal names to quantities that define hard eviction thresholds - // +kubebuilder:validation:XValidation:message="valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + // cpuManagerPolicy is the name of the policy to use. + // +kubebuilder:validation:Enum:={none,static} + // +kubebuilder:default="none" // +optional - EvictionHard map[string]string `json:"evictionHard,omitempty"` - // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds - // +kubebuilder:validation:XValidation:message="valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" - // +optional - EvictionSoft map[string]string `json:"evictionSoft,omitempty"` - // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal - // +kubebuilder:validation:XValidation:message="valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']",rule="self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'])" + CPUManagerPolicy string `json:"cpuManagerPolicy,omitempty"` + // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // Note: AKS CustomKubeletConfig uses cpuCfsQuota (camelCase) + // +kubebuilder:default=true // +optional - EvictionSoftGracePeriod map[string]metav1.Duration `json:"evictionSoftGracePeriod,omitempty"` - // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in - // response to soft eviction thresholds being met. + CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` + // cpuCfsQuotaPeriod sets the CPU CFS quota period value, `cpu.cfs_period_us`. + // The value must be between 1 ms and 1 second, inclusive. + // Default: "100ms" // +optional - EvictionMaxPodGracePeriod *int32 `json:"evictionMaxPodGracePeriod,omitempty"` + // +kubebuilder:default="100ms" + // TODO: validation + CPUCFSQuotaPeriod metav1.Duration `json:"cpuCFSQuotaPeriod,omitempty"` // ImageGCHighThresholdPercent is the percent of disk usage after which image // garbage collection is always run. The percent is calculated by dividing this // field value by 100, so this field must be between 0 and 100, inclusive. // When specified, the value must be greater than ImageGCLowThresholdPercent. + // Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name // +kubebuilder:validation:Minimum:=0 // +kubebuilder:validation:Maximum:=100 // +optional @@ -119,13 +101,55 @@ type KubeletConfiguration struct { // The percent is calculated by dividing this field value by 100, // so the field value must be between 0 and 100, inclusive. // When specified, the value must be less than imageGCHighThresholdPercent + // Note: AKS CustomKubeletConfig does not have "Percent" in the field name // +kubebuilder:validation:Minimum:=0 // +kubebuilder:validation:Maximum:=100 // +optional ImageGCLowThresholdPercent *int32 `json:"imageGCLowThresholdPercent,omitempty"` - // CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + // topologyManagerPolicy is the name of the topology manager policy to use. + // Valid values include: + // + // - `restricted`: kubelet only allows pods with optimal NUMA node alignment for requested resources; + // - `best-effort`: kubelet will favor pods with NUMA alignment of CPU and device resources; + // - `none`: kubelet has no knowledge of NUMA alignment of a pod's CPU and device resources. + // - `single-numa-node`: kubelet only allows pods with a single NUMA alignment + // of CPU and device resources. + // + // +kubebuilder:validation:Enum:={restricted,best-effort,none,single-numa-node} + // +kubebuilder:default="none" // +optional - CPUCFSQuota *bool `json:"cpuCFSQuota,omitempty"` + TopologyManagerPolicy string `json:"topologyManagerPolicy,omitempty"` + // A comma separated whitelist of unsafe sysctls or sysctl patterns (ending in `*`). + // Unsafe sysctl groups are `kernel.shm*`, `kernel.msg*`, `kernel.sem`, `fs.mqueue.*`, + // and `net.*`. For example: "`kernel.msg*,net.ipv4.route.min_pmtu`" + // Default: [] + // TODO: validation + // +optional + AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` + // failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. + // Default: true + // kubebuilder:default:=true + // +optional + FailSwapOn *bool `json:"failSwapOn,omitempty"` + // containerLogMaxSize is a quantity defining the maximum size of the container log + // file before it is rotated. For example: "5Mi" or "256Ki". + // Default: "10Mi" + // AKS CustomKubeletConfig has containerLogMaxSizeMB (with units), defaults to 50 + // +kubebuilder:validation:Pattern=`^\d+(E|P|T|G|M|K|Ei|Pi|Ti|Gi|Mi|Ki)$` + // +kubebuilder:default="50Mi" + // +optional + ContainerLogMaxSize string `json:"containerLogMaxSize,omitempty"` + // containerLogMaxFiles specifies the maximum number of container log files that can be present for a container. + // Default: 5 + // +kubebuilder:validation:Minimum:=2 + // +kubebuilder:default=5 + // +optional + ContainerLogMaxFiles *int32 `json:"containerLogMaxFiles,omitempty"` + // podPidsLimit is the maximum number of PIDs in any pod. + // AKS CustomKubeletConfig uses PodMaxPids, int32 (!) + // Default: -1 + // +optional + PodPidsLimit *int64 `json:"podPidsLimit,omitempty"` } // TODO: add hashing support diff --git a/pkg/apis/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/v1alpha2/zz_generated.deepcopy.go index 2f79bbc1f..48d6d31f1 100644 --- a/pkg/apis/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha2/zz_generated.deepcopy.go @@ -22,8 +22,7 @@ package v1alpha2 import ( "github.com/awslabs/operatorpkg/status" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -126,6 +125,11 @@ func (in *AKSNodeClassSpec) DeepCopyInto(out *AKSNodeClassSpec) { *out = new(KubeletConfiguration) (*in).DeepCopyInto(*out) } + if in.MaxPods != nil { + in, out := &in.MaxPods, &out.MaxPods + *out = new(int32) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AKSNodeClassSpec. @@ -165,7 +169,7 @@ func (in *Image) DeepCopyInto(out *Image) { *out = *in if in.Requirements != nil { in, out := &in.Requirements, &out.Requirements - *out = make([]corev1.NodeSelectorRequirement, len(*in)) + *out = make([]v1.NodeSelectorRequirement, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -185,61 +189,12 @@ func (in *Image) DeepCopy() *Image { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = *in - if in.ClusterDNS != nil { - in, out := &in.ClusterDNS, &out.ClusterDNS - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.MaxPods != nil { - in, out := &in.MaxPods, &out.MaxPods - *out = new(int32) - **out = **in - } - if in.PodsPerCore != nil { - in, out := &in.PodsPerCore, &out.PodsPerCore - *out = new(int32) - **out = **in - } - if in.SystemReserved != nil { - in, out := &in.SystemReserved, &out.SystemReserved - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.KubeReserved != nil { - in, out := &in.KubeReserved, &out.KubeReserved - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionHard != nil { - in, out := &in.EvictionHard, &out.EvictionHard - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoft != nil { - in, out := &in.EvictionSoft, &out.EvictionSoft - *out = make(map[string]string, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionSoftGracePeriod != nil { - in, out := &in.EvictionSoftGracePeriod, &out.EvictionSoftGracePeriod - *out = make(map[string]v1.Duration, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.EvictionMaxPodGracePeriod != nil { - in, out := &in.EvictionMaxPodGracePeriod, &out.EvictionMaxPodGracePeriod - *out = new(int32) + if in.CPUCFSQuota != nil { + in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + *out = new(bool) **out = **in } + out.CPUCFSQuotaPeriod = in.CPUCFSQuotaPeriod if in.ImageGCHighThresholdPercent != nil { in, out := &in.ImageGCHighThresholdPercent, &out.ImageGCHighThresholdPercent *out = new(int32) @@ -250,11 +205,26 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = new(int32) **out = **in } - if in.CPUCFSQuota != nil { - in, out := &in.CPUCFSQuota, &out.CPUCFSQuota + if in.AllowedUnsafeSysctls != nil { + in, out := &in.AllowedUnsafeSysctls, &out.AllowedUnsafeSysctls + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.FailSwapOn != nil { + in, out := &in.FailSwapOn, &out.FailSwapOn *out = new(bool) **out = **in } + if in.ContainerLogMaxFiles != nil { + in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles + *out = new(int32) + **out = **in + } + if in.PodPidsLimit != nil { + in, out := &in.PodPidsLimit, &out.PodPidsLimit + *out = new(int64) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeletConfiguration. diff --git a/pkg/providers/imagefamily/azlinux.go b/pkg/providers/imagefamily/azlinux.go index eb20bebfc..4412f3b83 100644 --- a/pkg/providers/imagefamily/azlinux.go +++ b/pkg/providers/imagefamily/azlinux.go @@ -73,7 +73,7 @@ func (u AzureLinux) DefaultImages() []DefaultImageOutput { } // UserData returns the default userdata script for the image Family -func (u AzureLinux) UserData(kubeletConfig *v1alpha2.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u AzureLinux) UserData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, diff --git a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go index ca6335b8f..e72f69ac7 100644 --- a/pkg/providers/imagefamily/bootstrap/aksbootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/aksbootstrap.go @@ -24,7 +24,6 @@ import ( "strings" "text/template" - "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" "github.com/Azure/karpenter-provider-azure/pkg/utils" "github.com/blang/semver/v4" "github.com/samber/lo" @@ -558,18 +557,13 @@ func normalizeResourceGroupNameForLabel(resourceGroupName string) string { return truncated } -func KubeletConfigToMap(kubeletConfig *v1alpha2.KubeletConfiguration) map[string]string { +func KubeletConfigToMap(kubeletConfig *KubeletConfiguration) map[string]string { args := make(map[string]string) if kubeletConfig == nil { return args } - if kubeletConfig.MaxPods != nil { - args["--max-pods"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.MaxPods)) - } - if kubeletConfig.PodsPerCore != nil { - args["--pods-per-core"] = fmt.Sprintf("%d", ptr.Int32Value(kubeletConfig.PodsPerCore)) - } + args["--max-pods"] = fmt.Sprintf("%d", kubeletConfig.MaxPods) JoinParameterArgsToMap(args, "--system-reserved", kubeletConfig.SystemReserved, "=") JoinParameterArgsToMap(args, "--kube-reserved", kubeletConfig.KubeReserved, "=") JoinParameterArgsToMap(args, "--eviction-hard", kubeletConfig.EvictionHard, "<") diff --git a/pkg/providers/imagefamily/bootstrap/bootstrap.go b/pkg/providers/imagefamily/bootstrap/bootstrap.go index f23468a4b..678475fcd 100644 --- a/pkg/providers/imagefamily/bootstrap/bootstrap.go +++ b/pkg/providers/imagefamily/bootstrap/bootstrap.go @@ -19,13 +19,34 @@ package bootstrap import ( "github.com/Azure/karpenter-provider-azure/pkg/apis/v1alpha2" core "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type KubeletConfiguration struct { + v1alpha2.KubeletConfiguration + + // MaxPods is the maximum number of pods that can run on a worker node instance. + MaxPods int32 + + SystemReserved map[string]string + // KubeReserved contains resources reserved for Kubernetes system components. + KubeReserved map[string]string + // EvictionHard is the map of signal names to quantities that define hard eviction thresholds + EvictionHard map[string]string + // EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + EvictionSoft map[string]string + // EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + EvictionSoftGracePeriod map[string]metav1.Duration + // EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + // response to soft eviction thresholds being met. + EvictionMaxPodGracePeriod *int32 +} + // Options is the node bootstrapping parameters passed from Karpenter to the provisioning node type Options struct { ClusterName string ClusterEndpoint string - KubeletConfig *v1alpha2.KubeletConfiguration + KubeletConfig *KubeletConfiguration Taints []core.Taint `hash:"set"` Labels map[string]string `hash:"set"` CABundle *string diff --git a/pkg/providers/imagefamily/resolver.go b/pkg/providers/imagefamily/resolver.go index 7d3ad98ef..06b8a1dc7 100644 --- a/pkg/providers/imagefamily/resolver.go +++ b/pkg/providers/imagefamily/resolver.go @@ -43,7 +43,7 @@ type Resolver struct { // ImageFamily can be implemented to override the default logic for generating dynamic launch template parameters type ImageFamily interface { UserData( - kubeletConfig *v1alpha2.KubeletConfiguration, + kubeletConfig *bootstrap.KubeletConfiguration, taints []corev1.Taint, labels map[string]string, caBundle *string, @@ -100,14 +100,21 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1alpha2.AKSNodeClass, return template, nil } -func prepareKubeletConfiguration(instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass) *v1alpha2.KubeletConfiguration { - kubeletConfig := nodeClass.Spec.Kubelet - if kubeletConfig == nil { - kubeletConfig = &v1alpha2.KubeletConfiguration{} +func prepareKubeletConfiguration(instanceType *cloudprovider.InstanceType, nodeClass *v1alpha2.AKSNodeClass) *bootstrap.KubeletConfiguration { + kubeletConfig := &bootstrap.KubeletConfiguration{} + + if nodeClass.Spec.Kubelet != nil { + kubeletConfig.KubeletConfiguration = *nodeClass.Spec.Kubelet + } + + // TODO: make default maxpods dependent on CNI + if nodeClass.Spec.MaxPods != nil { + kubeletConfig.MaxPods = *nodeClass.Spec.MaxPods + } else { + kubeletConfig.MaxPods = consts.DefaultKubernetesMaxPods } - kubeletConfig.MaxPods = lo.ToPtr[int32](consts.DefaultKubernetesMaxPods) - // TODO: revisit computeResources and maxPods implementation + // TODO: revisit computeResources implementation kubeletConfig.KubeReserved = utils.StringMap(instanceType.Overhead.KubeReserved) kubeletConfig.SystemReserved = utils.StringMap(instanceType.Overhead.SystemReserved) kubeletConfig.EvictionHard = map[string]string{instancetype.MemoryAvailable: instanceType.Overhead.EvictionThreshold.Memory().String()} diff --git a/pkg/providers/imagefamily/ubuntu_2204.go b/pkg/providers/imagefamily/ubuntu_2204.go index 7a5d20ae0..64b4496e9 100644 --- a/pkg/providers/imagefamily/ubuntu_2204.go +++ b/pkg/providers/imagefamily/ubuntu_2204.go @@ -73,7 +73,7 @@ func (u Ubuntu2204) DefaultImages() []DefaultImageOutput { } // UserData returns the default userdata script for the image Family -func (u Ubuntu2204) UserData(kubeletConfig *v1alpha2.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { +func (u Ubuntu2204) UserData(kubeletConfig *bootstrap.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ *cloudprovider.InstanceType) bootstrap.Bootstrapper { return bootstrap.AKS{ Options: bootstrap.Options{ ClusterName: u.Options.ClusterName, diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index fbbdfb902..920a0321c 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -124,7 +124,7 @@ func NewInstanceType(ctx context.Context, sku *skewer.SKU, vmsize *skewer.VMSize Name: sku.GetName(), Requirements: computeRequirements(sku, vmsize, architecture, offerings, region), Offerings: offerings, - Capacity: computeCapacity(ctx, sku, kc, nodeClass), + Capacity: computeCapacity(ctx, sku, nodeClass), Overhead: &cloudprovider.InstanceTypeOverhead{ KubeReserved: KubeReservedResources(lo.Must(sku.VCPU()), lo.Must(sku.Memory())), SystemReserved: SystemReservedResources(), @@ -261,12 +261,12 @@ func getArchitecture(architecture string) string { return architecture // unrecognized } -func computeCapacity(ctx context.Context, sku *skewer.SKU, kc *v1alpha2.KubeletConfiguration, nodeClass *v1alpha2.AKSNodeClass) corev1.ResourceList { +func computeCapacity(ctx context.Context, sku *skewer.SKU, nodeClass *v1alpha2.AKSNodeClass) corev1.ResourceList { return corev1.ResourceList{ corev1.ResourceCPU: *cpu(sku), corev1.ResourceMemory: *memory(ctx, sku), corev1.ResourceEphemeralStorage: *ephemeralStorage(nodeClass), - corev1.ResourcePods: *pods(sku, kc), + corev1.ResourcePods: *pods(nodeClass), corev1.ResourceName("nvidia.com/gpu"): *gpuNvidiaCount(sku), } } @@ -308,18 +308,15 @@ func ephemeralStorage(nodeClass *v1alpha2.AKSNodeClass) *resource.Quantity { return resource.NewScaledQuantity(int64(lo.FromPtr(nodeClass.Spec.OSDiskSizeGB)), resource.Giga) } -func pods(sku *skewer.SKU, kc *v1alpha2.KubeletConfiguration) *resource.Quantity { +func pods(nc *v1alpha2.AKSNodeClass) *resource.Quantity { // TODO: fine-tune pods calc var count int64 switch { - case kc != nil && kc.MaxPods != nil: - count = int64(ptr.Int32Value(kc.MaxPods)) + case nc.Spec.MaxPods != nil: + count = int64(ptr.Int32Value(nc.Spec.MaxPods)) default: count = 110 } - if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 { - count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * cpu(sku).Value(), count}) - } return resources.Quantity(fmt.Sprint(count)) } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index d74d13266..091d9929b 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -25,7 +25,6 @@ import ( "net/http" "strings" "testing" - "time" "github.com/awslabs/operatorpkg/status" "github.com/blang/semver/v4" @@ -460,14 +459,6 @@ var _ = Describe("InstanceType Provider", func() { Context("Nodepool with KubeletConfig", func() { It("should support provisioning with kubeletConfig, computeResources and maxPods not specified", func() { nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ - PodsPerCore: lo.ToPtr(int32(110)), - EvictionSoft: map[string]string{ - instancetype.MemoryAvailable: "1Gi", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - instancetype.MemoryAvailable: {Duration: 10 * time.Second}, - }, - EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), CPUCFSQuota: lo.ToPtr(true), @@ -481,14 +472,11 @@ var _ = Describe("InstanceType Provider", func() { customData := ExpectDecodedCustomData(azureEnv) expectedFlags := map[string]string{ - "eviction-hard": "memory.available<750Mi", - "eviction-soft": "memory.available<1Gi", - "eviction-soft-grace-period": "memory.available=10s", - "max-pods": "250", - "pods-per-core": "110", - "image-gc-low-threshold": "20", - "image-gc-high-threshold": "30", - "cpu-cfs-quota": "true", + "eviction-hard": "memory.available<750Mi", + "image-gc-high-threshold": "30", + "image-gc-low-threshold": "20", + "cpu-cfs-quota": "true", + "max-pods": "250", } ExpectKubeletFlags(azureEnv, customData, expectedFlags) @@ -534,14 +522,6 @@ var _ = Describe("InstanceType Provider", func() { }) It("should support provisioning with kubeletConfig, computeResources and maxPods not specified", func() { nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ - PodsPerCore: lo.ToPtr(int32(110)), - EvictionSoft: map[string]string{ - instancetype.MemoryAvailable: "1Gi", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - instancetype.MemoryAvailable: {Duration: 10 * time.Second}, - }, - EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), CPUCFSQuota: lo.ToPtr(true), @@ -554,14 +534,11 @@ var _ = Describe("InstanceType Provider", func() { customData := ExpectDecodedCustomData(azureEnv) expectedFlags := map[string]string{ - "eviction-hard": "memory.available<750Mi", - "eviction-soft": "memory.available<1Gi", - "eviction-soft-grace-period": "memory.available=10s", - "max-pods": "250", - "pods-per-core": "110", - "image-gc-low-threshold": "20", - "image-gc-high-threshold": "30", - "cpu-cfs-quota": "true", + "eviction-hard": "memory.available<750Mi", + "max-pods": "250", + "image-gc-low-threshold": "20", + "image-gc-high-threshold": "30", + "cpu-cfs-quota": "true", } ExpectKubeletFlags(azureEnv, customData, expectedFlags) Expect(customData).To(SatisfyAny( // AKS default @@ -575,31 +552,11 @@ var _ = Describe("InstanceType Provider", func() { }) It("should support provisioning with kubeletConfig, computeResources and maxPods specified", func() { nodeClass.Spec.Kubelet = &v1alpha2.KubeletConfiguration{ - PodsPerCore: lo.ToPtr(int32(110)), - EvictionSoft: map[string]string{ - instancetype.MemoryAvailable: "1Gi", - }, - EvictionSoftGracePeriod: map[string]metav1.Duration{ - instancetype.MemoryAvailable: {Duration: 10 * time.Second}, - }, - EvictionMaxPodGracePeriod: lo.ToPtr(int32(15)), ImageGCHighThresholdPercent: lo.ToPtr(int32(30)), ImageGCLowThresholdPercent: lo.ToPtr(int32(20)), CPUCFSQuota: lo.ToPtr(true), - - SystemReserved: map[string]string{ - string(v1.ResourceCPU): "200m", - string(v1.ResourceMemory): "1Gi", - }, - KubeReserved: map[string]string{ - string(v1.ResourceCPU): "100m", - string(v1.ResourceMemory): "500Mi", - }, - EvictionHard: map[string]string{ - instancetype.MemoryAvailable: "10Mi", - }, - MaxPods: lo.ToPtr(int32(15)), } + nodeClass.Spec.MaxPods = lo.ToPtr(int32(15)) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -608,14 +565,11 @@ var _ = Describe("InstanceType Provider", func() { customData := ExpectDecodedCustomData(azureEnv) expectedFlags := map[string]string{ - "eviction-hard": "memory.available<750Mi", - "eviction-soft": "memory.available<1Gi", - "eviction-soft-grace-period": "memory.available=10s", - "max-pods": "250", // max pods should always default to 250 - "pods-per-core": "110", - "image-gc-low-threshold": "20", - "image-gc-high-threshold": "30", - "cpu-cfs-quota": "true", + "eviction-hard": "memory.available<750Mi", + "max-pods": "15", + "image-gc-low-threshold": "20", + "image-gc-high-threshold": "30", + "cpu-cfs-quota": "true", } ExpectKubeletFlags(azureEnv, customData, expectedFlags) From efbe216fdf234c2ce12bacf22c8428b5bd5e0349 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Wed, 16 Oct 2024 21:30:06 +0000 Subject: [PATCH 36/47] fix: conflicting nodeclaim.garbagecollcation controller name --- pkg/controllers/nodeclaim/garbagecollection/controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index e3a5bc93c..033dc31f3 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -56,7 +56,7 @@ func NewController(kubeClient client.Client, cloudProvider corecloudprovider.Clo } func (c *Controller) Reconcile(ctx context.Context) (reconcile.Result, error) { - ctx = injection.WithControllerName(ctx, "nodeclaim.garbagecollection") + ctx = injection.WithControllerName(ctx, "instance.garbagecollection") // We LIST VMs on the CloudProvider BEFORE we grab NodeClaims/Nodes on the cluster so that we make sure that, if // LISTing instances takes a long time, our information is more updated by the time we get to nodeclaim and Node LIST @@ -114,7 +114,7 @@ func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *karpv1.NodeC func (c *Controller) Register(_ context.Context, m manager.Manager) error { return controllerruntime.NewControllerManagedBy(m). - Named("nodeclaim.garbagecollection"). + Named("instance.garbagecollection"). WatchesRawSource(singleton.Source()). Complete(singleton.AsReconciler(c)) } From c27a1cef8439e916d072180f5c85dd4bd3f7c66b Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Fri, 18 Oct 2024 00:53:51 +0000 Subject: [PATCH 37/47] chore: restore webhooks in alt operator --- cmd/controller/main_ccp.go | 3 ++- pkg/alt/karpenter-core/pkg/operator/operator.go | 13 +++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 91401209c..91c1a4559 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -28,6 +28,7 @@ import ( controllers "github.com/Azure/karpenter-provider-azure/pkg/controllers" "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" + corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" // Note the absence of corewebhooks: these pull in knative webhook-related packages and informers in init() // We don't give cluster-level roles when running in AKS managed mode, so their informers will produce errors and halt all other operations // corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" @@ -56,7 +57,7 @@ func main() { op.EventRecorder, cloudProvider, )...). - // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). + WithWebhooks(ctx, corewebhooks.NewWebhooks()...). WithControllers(ctx, controllers.NewControllers( ctx, op.Manager, diff --git a/pkg/alt/karpenter-core/pkg/operator/operator.go b/pkg/alt/karpenter-core/pkg/operator/operator.go index 19c3266f7..86c5369b4 100644 --- a/pkg/alt/karpenter-core/pkg/operator/operator.go +++ b/pkg/alt/karpenter-core/pkg/operator/operator.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -// Source: https://github.com/kubernetes-sigs/karpenter/blob/v0.30.0/pkg/operator/operator.go +// Source: https://github.com/kubernetes-sigs/karpenter/blob/v1.0.4/pkg/operator/operator.go package operator @@ -51,6 +51,8 @@ import ( knativeinjection "knative.dev/pkg/injection" "knative.dev/pkg/signals" "knative.dev/pkg/system" + "knative.dev/pkg/webhook" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" @@ -74,11 +76,9 @@ const ( // Source: NewOperator() // Modified behavior: // - Allow Karpenter and most components to exist on control plane, but can reach the CRs on overlay -// - Webhooks not supported // - Karpenter will not crash if CRDs are not found, but goes into a retry loop for a while // Modified implementations: // - Split the context into two: control plane and overlay -// - Remove webhooks-related code // - Retry loop for getting CRDs // - Introduce and retrieve overlay namespace from env // - No profiling @@ -112,7 +112,12 @@ func NewOperator() (context.Context, *coreoperator.Operator) { klog.SetLogger(logger) // Webhook - // Unsupported -- skipping + overlayCtx = webhook.WithOptions(overlayCtx, webhook.Options{ + Port: options.FromContext(overlayCtx).WebhookPort, + ServiceName: options.FromContext(overlayCtx).ServiceName, + SecretName: fmt.Sprintf("%s-cert", options.FromContext(overlayCtx).ServiceName), + GracePeriod: 5 * time.Second, + }) // Client Config ccPlaneConfig := lo.Must(rest.InClusterConfig()) From 16d2984e7bed955ca0103d245a1ed99d5f03f6a5 Mon Sep 17 00:00:00 2001 From: Matthew Christopher Date: Fri, 18 Oct 2024 03:23:05 +0000 Subject: [PATCH 38/47] Clean up commented out webhook code --- cmd/controller/main_ccp.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 91c1a4559..540cd2ebe 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -29,9 +29,6 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" - // Note the absence of corewebhooks: these pull in knative webhook-related packages and informers in init() - // We don't give cluster-level roles when running in AKS managed mode, so their informers will produce errors and halt all other operations - // corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" ) func main() { @@ -66,6 +63,5 @@ func main() { aksCloudProvider, op.InstanceProvider, )...). - // WithWebhooks(ctx, corewebhooks.NewWebhooks()...). Start(ctx, cloudProvider) } From d0b074a299dcfc4e02da5d75d135537584180980 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Tue, 22 Oct 2024 06:53:14 +0000 Subject: [PATCH 39/47] fix(test): fix test for credential provider URL in custom data --- pkg/providers/instancetype/suite_test.go | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 091d9929b..c0e30f560 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -19,7 +19,6 @@ package instancetype_test import ( "bytes" "context" - "encoding/base64" "fmt" "io" "net/http" @@ -1078,7 +1077,7 @@ var _ = Describe("InstanceType Provider", func() { Context("Bootstrap", func() { var ( kubeletFlags string - decodedString string + customData string minorVersion uint64 credentialProviderURL string ) @@ -1087,7 +1086,8 @@ var _ = Describe("InstanceType Provider", func() { pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, coreProvisioner, pod) ExpectScheduled(ctx, env.Client, pod) - kubeletFlags = ExpectKubeletFlagsPassed() + customData = ExpectDecodedCustomData(azureEnv) + kubeletFlags = ExpectKubeletFlagsPassed(customData) k8sVersion, err := azureEnv.ImageProvider.KubeServerVersion(ctx) Expect(err).To(BeNil()) @@ -1108,7 +1108,7 @@ var _ = Describe("InstanceType Provider", func() { Expect(kubeletFlags).ToNot(ContainSubstring("--azure-container-registry-config")) Expect(kubeletFlags).To(ContainSubstring("--image-credential-provider-config=/var/lib/kubelet/credential-provider-config.yaml")) Expect(kubeletFlags).To(ContainSubstring("--image-credential-provider-bin-dir=/var/lib/kubelet/credential-provider")) - Expect(decodedString).To(ContainSubstring(credentialProviderURL)) + Expect(customData).To(ContainSubstring(credentialProviderURL)) } }) @@ -1252,14 +1252,8 @@ func createSDKErrorBody(code, message string) io.ReadCloser { return io.NopCloser(bytes.NewReader([]byte(fmt.Sprintf(`{"error":{"code": "%s", "message": "%s"}}`, code, message)))) } -func ExpectKubeletFlagsPassed() string { +func ExpectKubeletFlagsPassed(customData string) string { GinkgoHelper() - Expect(azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Len()).To(Equal(1)) - vm := azureEnv.VirtualMachinesAPI.VirtualMachineCreateOrUpdateBehavior.CalledWithInput.Pop().VM - customData := *vm.Properties.OSProfile.CustomData - Expect(customData).ToNot(BeNil()) - decodedBytes, err := base64.StdEncoding.DecodeString(customData) - Expect(err).To(Succeed()) - decodedString := string(decodedBytes[:]) - return decodedString[strings.Index(decodedString, "KUBELET_FLAGS=")+len("KUBELET_FLAGS=") : strings.Index(decodedString, "KUBELET_NODE_LABELS")] + + return customData[strings.Index(customData, "KUBELET_FLAGS=")+len("KUBELET_FLAGS=") : strings.Index(customData, "KUBELET_NODE_LABELS")] } From ea188ff8c8051cd95be7416cda9ab7f5dd3eb6b7 Mon Sep 17 00:00:00 2001 From: Matthew Christopher Date: Wed, 23 Oct 2024 10:46:26 -0700 Subject: [PATCH 40/47] Make webhooks work in AKS CCP context (#537) This requires quite a bit of hacking, mostly overriding certain things in the ctx. The major items are: * Copy and modify knative/pkg/webhook/resourcesemantics/conversion to support CRD clientConfig.url in addition to clientConfig.service. * Copy and modify karpenter/pkg/webhooks/webhooks.go to support overriding the informer factory, so that we can point it at the CCP APIServer rather than overlay. * Override Start and supporting methods on the provider specific operator in pkg/operator/operator.go to allow invoking our modified version of karpenter/pkg/webhooks/webhooks.go. --- .golangci.yaml | 2 + cmd/controller/main_ccp.go | 45 +- go.mod | 5 +- .../karpenter-core/pkg/operator/operator.go | 30 +- pkg/alt/karpenter-core/pkg/webhooks/README.md | 4 + .../karpenter-core/pkg/webhooks/webhooks.go | 205 ++++++ .../resourcesemantics/conversion/README.md | 10 + .../conversion/controller.go | 159 +++++ .../conversion/conversion.go | 206 ++++++ .../conversion/conversion_test.go | 630 ++++++++++++++++++ .../conversion/internal/types.go | 278 ++++++++ .../internal/zz_generated.deepcopy.go | 132 ++++ .../resourcesemantics/conversion/options.go | 46 ++ .../conversion/options_test.go | 38 ++ .../conversion/reconciler.go | 123 ++++ .../conversion/table_test.go | 290 ++++++++ pkg/operator/operator.go | 59 ++ 17 files changed, 2255 insertions(+), 7 deletions(-) create mode 100644 pkg/alt/karpenter-core/pkg/webhooks/README.md create mode 100644 pkg/alt/karpenter-core/pkg/webhooks/webhooks.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go create mode 100644 pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go diff --git a/.golangci.yaml b/.golangci.yaml index 3049d52db..039b490e7 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -80,5 +80,7 @@ issues: - hack - charts - designs + - pkg/alt/knative # copy + - pkg/alt/karpenter-core/pkg/webhooks # copy exclude-files: - pkg/alt/karpenter-core/pkg/operator/logger.go # copy diff --git a/cmd/controller/main_ccp.go b/cmd/controller/main_ccp.go index 540cd2ebe..6d95d2882 100644 --- a/cmd/controller/main_ccp.go +++ b/cmd/controller/main_ccp.go @@ -19,18 +19,55 @@ limitations under the License. package main import ( + "context" + "github.com/samber/lo" + "go.uber.org/zap" + "knative.dev/pkg/logging" - "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" - "github.com/Azure/karpenter-provider-azure/pkg/operator" + // Injection stuff + kubeclientinjection "knative.dev/pkg/client/injection/kube/client" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + knativeinjection "knative.dev/pkg/injection" + secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret" + kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" + "knative.dev/pkg/webhook/certificates" altOperator "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/operator" + altwebhooks "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/webhooks" + "github.com/Azure/karpenter-provider-azure/pkg/cloudprovider" controllers "github.com/Azure/karpenter-provider-azure/pkg/controllers" + "github.com/Azure/karpenter-provider-azure/pkg/operator" "sigs.k8s.io/karpenter/pkg/cloudprovider/metrics" corecontrollers "sigs.k8s.io/karpenter/pkg/controllers" - corewebhooks "sigs.k8s.io/karpenter/pkg/webhooks" ) +func newWebhooks(ctx context.Context) []knativeinjection.ControllerConstructor { + client := altOperator.GetCCPClient(ctx) + ccpInformerFactory := kubeinformerfactory.Get(ctx) + + secretInformer := ccpInformerFactory.Core().V1().Secrets() + ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) + + logging.FromContext(ctx).Info("Starting horrible CCP informer") + if err := controller.StartInformers(ctx.Done(), secretInformer.Informer()); err != nil { + logging.FromContext(ctx).Fatalw("Failed to start horrible CCP informer", zap.Error(err)) + } + + return []knativeinjection.ControllerConstructor{ + func(ctx context.Context, watcher configmap.Watcher) *controller.Impl { + ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) + ctx = context.WithValue(ctx, kubeclientinjection.Key{}, client) + return certificates.NewController(ctx, watcher) + }, + func(ctx context.Context, watcher configmap.Watcher) *controller.Impl { + ctx = context.WithValue(ctx, secretinformer.Key{}, secretInformer) + return altwebhooks.NewCRDConversionWebhook(ctx, watcher) + }, + } +} + func main() { //ctx, op := operator.NewOperator(coreoperator.NewOperator()) ctx, op := operator.NewOperator(altOperator.NewOperator()) @@ -54,7 +91,7 @@ func main() { op.EventRecorder, cloudProvider, )...). - WithWebhooks(ctx, corewebhooks.NewWebhooks()...). + WithWebhooks(ctx, newWebhooks(ctx)...). WithControllers(ctx, controllers.NewControllers( ctx, op.Manager, diff --git a/go.mod b/go.mod index ffd492a7a..434cd7824 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/go-logr/logr v1.4.2 github.com/go-logr/zapr v1.3.0 github.com/go-playground/validator/v10 v10.22.1 + github.com/google/go-cmp v0.6.0 github.com/imdario/mergo v0.3.16 github.com/jongio/azidext/go/azidext v0.5.0 github.com/mitchellh/hashstructure/v2 v2.0.2 @@ -33,6 +34,7 @@ require ( github.com/stretchr/testify v1.9.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.27.0 + golang.org/x/sync v0.8.0 k8s.io/api v0.30.3 k8s.io/apiextensions-apiserver v0.30.3 k8s.io/apimachinery v0.30.3 @@ -73,6 +75,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch v5.9.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect @@ -92,7 +95,6 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20240827171923-fa2c70bbbfe5 // indirect github.com/google/uuid v1.6.0 // indirect @@ -131,7 +133,6 @@ require ( golang.org/x/exp v0.0.0-20240808152545-0cdaa3abc0fa // indirect golang.org/x/net v0.29.0 // indirect golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect diff --git a/pkg/alt/karpenter-core/pkg/operator/operator.go b/pkg/alt/karpenter-core/pkg/operator/operator.go index 86c5369b4..c75c8b59d 100644 --- a/pkg/alt/karpenter-core/pkg/operator/operator.go +++ b/pkg/alt/karpenter-core/pkg/operator/operator.go @@ -42,13 +42,16 @@ import ( "sigs.k8s.io/karpenter/pkg/operator/injection" "sigs.k8s.io/karpenter/pkg/operator/options" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/util/flowcontrol" "k8s.io/utils/clock" + "knative.dev/pkg/controller" knativeinjection "knative.dev/pkg/injection" + kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" "knative.dev/pkg/signals" "knative.dev/pkg/system" "knative.dev/pkg/webhook" @@ -73,6 +76,20 @@ const ( component = "controller" ) +type ccpClient struct{} + +func withCCPClient(ctx context.Context, client kubernetes.Interface) context.Context { + return context.WithValue(ctx, ccpClient{}, client) +} + +func GetCCPClient(ctx context.Context) kubernetes.Interface { + v := ctx.Value(ccpClient{}) + if v == nil { + return nil + } + return v.(kubernetes.Interface) +} + // Source: NewOperator() // Modified behavior: // - Allow Karpenter and most components to exist on control plane, but can reach the CRs on overlay @@ -112,7 +129,7 @@ func NewOperator() (context.Context, *coreoperator.Operator) { klog.SetLogger(logger) // Webhook - overlayCtx = webhook.WithOptions(overlayCtx, webhook.Options{ + ccPlaneCtx = webhook.WithOptions(ccPlaneCtx, webhook.Options{ Port: options.FromContext(overlayCtx).WebhookPort, ServiceName: options.FromContext(overlayCtx).ServiceName, SecretName: fmt.Sprintf("%s-cert", options.FromContext(overlayCtx).ServiceName), @@ -130,6 +147,17 @@ func NewOperator() (context.Context, *coreoperator.Operator) { // Client overlayKubernetesInterface := kubernetes.NewForConfigOrDie(overlayConfig) + ccpKubernetesInterface := kubernetes.NewForConfigOrDie(ccPlaneConfig) + ccPlaneCtx = withCCPClient(ccPlaneCtx, ccpKubernetesInterface) + + ccpInformerFactory := informers.NewSharedInformerFactoryWithOptions( + ccpKubernetesInterface, + controller.GetResyncPeriod(ccPlaneCtx), + // This factory scopes things to the system namespace. + informers.WithNamespace(system.Namespace())) + // Also override the kubeinformerfactory because it's used by webhook.New() to construct + // a new secret client, see: https://github.com/knative/pkg/blob/main/webhook/webhook.go#L183 + ccPlaneCtx = context.WithValue(ccPlaneCtx, kubeinformerfactory.Key{}, ccpInformerFactory) // Manager mgrOpts := ctrl.Options{ diff --git a/pkg/alt/karpenter-core/pkg/webhooks/README.md b/pkg/alt/karpenter-core/pkg/webhooks/README.md new file mode 100644 index 000000000..6236d4dd8 --- /dev/null +++ b/pkg/alt/karpenter-core/pkg/webhooks/README.md @@ -0,0 +1,4 @@ +This is copied from https://github.com/kubernetes-sigs/karpenter/blob/v1.0.4/pkg/webhooks/webhooks.go + +Some modifications have been made to cater to the deployment model in AKS (multiple API servers). +Look for the sections that start with `// AKS customized` to understand the changes. diff --git a/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go b/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go new file mode 100644 index 000000000..6fef858b2 --- /dev/null +++ b/pkg/alt/karpenter-core/pkg/webhooks/webhooks.go @@ -0,0 +1,205 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package webhooks + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/Azure/karpenter-provider-azure/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion" + "github.com/awslabs/operatorpkg/object" + "github.com/samber/lo" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" + "k8s.io/apimachinery/pkg/runtime/schema" + informers "k8s.io/client-go/informers" + "k8s.io/client-go/rest" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + knativeinjection "knative.dev/pkg/injection" + kubeinformerfactory "knative.dev/pkg/injection/clients/namespacedkube/informers/factory" + "knative.dev/pkg/injection/sharedmain" + knativelogging "knative.dev/pkg/logging" + "knative.dev/pkg/metrics" + "knative.dev/pkg/webhook" + "knative.dev/pkg/webhook/certificates" + "sigs.k8s.io/controller-runtime/pkg/healthz" + + v1 "sigs.k8s.io/karpenter/pkg/apis/v1" + "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/operator/injection" + "sigs.k8s.io/karpenter/pkg/operator/logging" + "sigs.k8s.io/karpenter/pkg/operator/options" +) + +const component = "webhook" + +var ( + // TODO: Remove conversion webhooks once support for the v1beta1 APIs is dropped + ConversionResource = map[schema.GroupKind]conversion.GroupKindConversion{ + object.GVK(&v1beta1.NodePool{}).GroupKind(): { + DefinitionName: "nodepools.karpenter.sh", + HubVersion: "v1", + Zygotes: map[string]conversion.ConvertibleObject{ + "v1": &v1.NodePool{}, + "v1beta1": &v1beta1.NodePool{}, + }, + }, + object.GVK(&v1beta1.NodeClaim{}).GroupKind(): { + DefinitionName: "nodeclaims.karpenter.sh", + HubVersion: "v1", + Zygotes: map[string]conversion.ConvertibleObject{ + "v1": &v1.NodeClaim{}, + "v1beta1": &v1beta1.NodeClaim{}, + }, + }, + } +) + +func NewWebhooks() []knativeinjection.ControllerConstructor { + return []knativeinjection.ControllerConstructor{ + certificates.NewController, + NewCRDConversionWebhook, + } +} + +func NewCRDConversionWebhook(ctx context.Context, _ configmap.Watcher) *controller.Impl { + nodeclassCtx := injection.GetNodeClasses(ctx) + client := injection.GetClient(ctx) + return conversion.NewConversionController( + ctx, + "/conversion/karpenter.sh", + ConversionResource, + func(ctx context.Context) context.Context { + return injection.WithClient(injection.WithNodeClasses(ctx, nodeclassCtx), client) + }, + ) +} + +// Start copies the relevant portions for starting the webhooks from sharedmain.MainWithConfig +// https://github.com/knative/pkg/blob/0f52db700d63/injection/sharedmain/main.go#L227 +func Start(ctx context.Context, cfg *rest.Config, ctors ...knativeinjection.ControllerConstructor) { + logger := logging.NewLogger(ctx, component).Sugar() + ctx = knativelogging.WithLogger(ctx, logger) + + // AKS customized + // This section is customized for AKS use-case to allow overriding the informer factory. + // Overriding the informer factory is required to make webhooks work, as it is used + // in webhook.New() to construct a new secret client, see: + // https://github.com/knative/pkg/blob/main/webhook/webhook.go#L183 + var overriddenInformerFactory informers.SharedInformerFactory + if ctx.Value(kubeinformerfactory.Key{}) != nil { + overriddenInformerFactory = kubeinformerfactory.Get(ctx) + } + // end customization + + ctx, startInformers := knativeinjection.EnableInjectionOrDie(ctx, cfg) + cmw := sharedmain.SetupConfigMapWatchOrDie(ctx, logger) + controllers, webhooks := sharedmain.ControllersAndWebhooksFromCtors(ctx, cmw, ctors...) + + // AKS customized + if overriddenInformerFactory != nil { + ctx = context.WithValue(ctx, kubeinformerfactory.Key{}, overriddenInformerFactory) + } + // end customization + + // Many of the webhooks rely on configuration, e.g. configurable defaults, feature flags. + // So make sure that we have synchronized our configuration state before launching the + // webhooks, so that things are properly initialized. + logger.Info("Starting configuration manager...") + if err := cmw.Start(ctx.Done()); err != nil { + logger.Fatalw("Failed to start configuration manager", zap.Error(err)) + } + + // If we have one or more admission controllers, then start the webhook + // and pass them in. + var wh *webhook.Webhook + var err error + eg, egCtx := errgroup.WithContext(ctx) + if len(webhooks) > 0 { + // Update the metric exporter to point to a prometheus endpoint + lo.Must0(metrics.UpdateExporter(ctx, metrics.ExporterOptions{ + Component: strings.ReplaceAll(component, "-", "_"), + ConfigMap: lo.Must(metrics.NewObservabilityConfigFromConfigMap(nil)).GetConfigMap().Data, + Secrets: sharedmain.SecretFetcher(ctx), + PrometheusPort: options.FromContext(ctx).WebhookMetricsPort, + }, logger)) + // Register webhook metrics + webhook.RegisterMetrics() + + wh, err = webhook.New(ctx, webhooks) + if err != nil { + logger.Fatalw("Failed to create webhook", zap.Error(err)) + } + eg.Go(func() error { + return wh.Run(ctx.Done()) + }) + } + + // Start the injection clients and informers. + startInformers() + + // Wait for webhook informers to sync. + if wh != nil { + wh.InformersHaveSynced() + } + logger.Info("Starting controllers...") + eg.Go(func() error { + return controller.StartAll(ctx, controllers...) + }) + // This will block until either a signal arrives or one of the grouped functions + // returns an error. + <-egCtx.Done() + + // Don't forward ErrServerClosed as that indicates we're already shutting down. + if err := eg.Wait(); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Errorw("Error while running server", zap.Error(err)) + } +} + +func HealthProbe(ctx context.Context) healthz.Checker { + // Create new transport that doesn't validate the TLS certificate + // This transport is just polling so validating the server certificate isn't necessary + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} // nolint:gosec + client := &http.Client{Transport: transport} + + // TODO: Add knative health check port for webhooks when health port can be configured + // Issue: https://github.com/knative/pkg/issues/2765 + return func(req *http.Request) (err error) { + res, err := client.Get(fmt.Sprintf("https://localhost:%d", options.FromContext(ctx).WebhookPort)) + // If the webhook connection errors out, liveness/readiness should fail + if err != nil { + return err + } + // Close the body to avoid leaking file descriptors + // Always read the body so we can re-use the connection: https://stackoverflow.com/questions/17948827/reusing-http-connections-in-go + _, _ = io.ReadAll(res.Body) + res.Body.Close() + + // If there is a server-side error or path not found, + // consider liveness to have failed + if res.StatusCode >= 500 || res.StatusCode == 404 { + return fmt.Errorf("webhook probe failed with status code %d", res.StatusCode) + } + return nil + } +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md new file mode 100644 index 000000000..7424c3768 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/README.md @@ -0,0 +1,10 @@ +This code is all copied from https://github.com/knative/pkg/tree/main/webhook/resourcesemantics/conversion +at commit fdbc0b5adde7 (this is the commit used by our knative dependency v0.0.0-20240910170930-fdbc0b5adde7 +as of Oct 23, 2024). + +The only changes were made to reconciler.go to support `crd.spec.conversion.webhook.clientConfig.url` in addition +to `crd.spec.conversion.webhook.clientConfig.service`. + +Look for the sections that start with `// AKS customized` to understand the changes. + +Also note that the zz_generated.deepcopy.go isn't currently actually autogenerated it's just copied from upstream. diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go new file mode 100644 index 000000000..fbb9a8e2c --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/controller.go @@ -0,0 +1,159 @@ +// Portions Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package conversion + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/cache" + "knative.dev/pkg/apis" + apixclient "knative.dev/pkg/client/injection/apiextensions/client" + crdinformer "knative.dev/pkg/client/injection/apiextensions/informers/apiextensions/v1/customresourcedefinition" + "knative.dev/pkg/controller" + secretinformer "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret" + "knative.dev/pkg/logging" + pkgreconciler "knative.dev/pkg/reconciler" + "knative.dev/pkg/system" + "knative.dev/pkg/webhook" +) + +// ConvertibleObject defines the functionality our API types +// are required to implement in order to be convertible from +// one version to another +// +// Optionally if the object implements apis.Defaultable the +// ConversionController will apply defaults before returning +// the response +type ConvertibleObject interface { + // ConvertTo(ctx, to) + // ConvertFrom(ctx, from) + apis.Convertible + + // DeepCopyObject() + // GetObjectKind() => SetGroupVersionKind(gvk) + runtime.Object +} + +// GroupKindConversion specifies how a specific Kind for a given +// group should be converted +type GroupKindConversion struct { + // DefinitionName specifies the CustomResourceDefinition that should + // be reconciled with by the controller. + // + // The conversion webhook configuration will be updated + // when the CA bundle changes + DefinitionName string + + // HubVersion specifies which version of the CustomResource supports + // conversions to and from all types + // + // It is expected that the Zygotes map contains an entry for the + // specified HubVersion + HubVersion string + + // Zygotes contains a map of version strings (ie. v1, v2) to empty + // ConvertibleObject objects + // + // During a conversion request these zygotes will be deep copied + // and manipulated using the apis.Convertible interface + Zygotes map[string]ConvertibleObject +} + +// NewConversionController returns a K8s controller that will +// will reconcile CustomResourceDefinitions and update their +// conversion webhook attributes such as path & CA bundle. +// +// Additionally the controller's Reconciler implements +// webhook.ConversionController for the purposes of converting +// resources between different versions +func NewConversionController( + ctx context.Context, + path string, + kinds map[schema.GroupKind]GroupKindConversion, + withContext func(context.Context) context.Context, +) *controller.Impl { + opts := []OptionFunc{ + WithPath(path), + WithWrapContext(withContext), + WithKinds(kinds), + } + + return newController(ctx, opts...) +} + +func newController(ctx context.Context, optsFunc ...OptionFunc) *controller.Impl { + secretInformer := secretinformer.Get(ctx) + crdInformer := crdinformer.Get(ctx) + client := apixclient.Get(ctx) + woptions := webhook.GetOptions(ctx) + + opts := &options{} + + for _, f := range optsFunc { + f(opts) + } + + r := &reconciler{ + LeaderAwareFuncs: pkgreconciler.LeaderAwareFuncs{ + // Have this reconciler enqueue our types whenever it becomes leader. + PromoteFunc: func(bkt pkgreconciler.Bucket, enq func(pkgreconciler.Bucket, types.NamespacedName)) error { + for _, gkc := range opts.kinds { + name := gkc.DefinitionName + enq(bkt, types.NamespacedName{Name: name}) + } + return nil + }, + }, + + kinds: opts.kinds, + path: opts.path, + secretName: woptions.SecretName, + withContext: opts.wc, + + client: client, + secretLister: secretInformer.Lister(), + crdLister: crdInformer.Lister(), + } + + logger := logging.FromContext(ctx) + controllerOptions := woptions.ControllerOptions + if controllerOptions == nil { + const queueName = "ConversionWebhook" + controllerOptions = &controller.ControllerOptions{WorkQueueName: queueName, Logger: logger.Named(queueName)} + } + c := controller.NewContext(ctx, r, *controllerOptions) + + // Reconciler when the named CRDs change. + for _, gkc := range opts.kinds { + name := gkc.DefinitionName + + crdInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.FilterWithName(name), + Handler: controller.HandleAll(c.Enqueue), + }) + + sentinel := c.EnqueueSentinel(types.NamespacedName{Name: name}) + + // Reconcile when the cert bundle changes. + secretInformer.Informer().AddEventHandler(cache.FilteringResourceEventHandler{ + FilterFunc: controller.FilterWithNameAndNamespace(system.Namespace(), woptions.SecretName), + Handler: controller.HandleAll(sentinel), + }) + } + + return c +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go new file mode 100644 index 000000000..761acf81c --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion.go @@ -0,0 +1,206 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "encoding/json" + "fmt" + + "go.uber.org/zap" + + apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + + "knative.dev/pkg/apis" + "knative.dev/pkg/kmeta" + "knative.dev/pkg/logging" + "knative.dev/pkg/logging/logkey" +) + +// Convert implements webhook.ConversionController +func (r *reconciler) Convert( + ctx context.Context, + req *apixv1.ConversionRequest, +) *apixv1.ConversionResponse { + if r.withContext != nil { + ctx = r.withContext(ctx) + } + + res := &apixv1.ConversionResponse{ + UID: req.UID, + Result: metav1.Status{ + Status: metav1.StatusSuccess, + }, + } + + result := make([]runtime.RawExtension, 0, len(req.Objects)) + + for _, obj := range req.Objects { + converted, err := r.convert(ctx, obj, req.DesiredAPIVersion) + if err != nil { + logging.FromContext(ctx).Errorw("Conversion failed", zap.Error(err)) + res.Result.Status = metav1.StatusFailure + res.Result.Message = err.Error() + break + } + + result = append(result, converted) + } + + res.ConvertedObjects = result + return res +} + +func (r *reconciler) convert( + ctx context.Context, + inRaw runtime.RawExtension, + targetVersion string, +) (runtime.RawExtension, error) { + logger := logging.FromContext(ctx) + var ret runtime.RawExtension + + inGVK, err := parseGVK(inRaw) + if err != nil { + return ret, err + } + + inGK := inGVK.GroupKind() + conv, ok := r.kinds[inGK] + if !ok { + return ret, fmt.Errorf("no conversion support for type %s", formatGK(inGVK.GroupKind())) + } + + outGVK, err := parseAPIVersion(targetVersion, inGK.Kind) + if err != nil { + return ret, err + } + + inZygote, ok := conv.Zygotes[inGVK.Version] + if !ok { + return ret, fmt.Errorf("conversion not supported for type %s", formatGVK(inGVK)) + } + outZygote, ok := conv.Zygotes[outGVK.Version] + if !ok { + return ret, fmt.Errorf("conversion not supported for type %s", formatGVK(outGVK)) + } + hubZygote, ok := conv.Zygotes[conv.HubVersion] + if !ok { + return ret, fmt.Errorf("conversion not supported for type %s", formatGK(inGVK.GroupKind())) + } + + in := inZygote.DeepCopyObject().(ConvertibleObject) + hub := hubZygote.DeepCopyObject().(ConvertibleObject) + out := outZygote.DeepCopyObject().(ConvertibleObject) + + hubGVK := inGVK.GroupKind().WithVersion(conv.HubVersion) + + logger = logger.With( + zap.String("inputType", formatGVK(inGVK)), + zap.String("outputType", formatGVK(outGVK)), + zap.String("hubType", formatGVK(hubGVK)), + ) + + // TODO(dprotaso) - potentially error on unknown fields + if err = json.Unmarshal(inRaw.Raw, &in); err != nil { + return ret, fmt.Errorf("unable to unmarshal input: %w", err) + } + + if acc, err := kmeta.DeletionHandlingAccessor(in); err == nil { + // TODO: right now we don't convert any non-namespaced objects. If we ever do that + // this needs to updated to deal with it. + logger = logger.With(zap.String(logkey.Key, acc.GetNamespace()+"/"+acc.GetName())) + } else { + logger.Infof("Could not get Accessor for %s: %v", formatGK(inGVK.GroupKind()), err) + } + ctx = logging.WithLogger(ctx, logger) + + if inGVK.Version == conv.HubVersion { + hub = in + } else if err = hub.ConvertFrom(ctx, in); err != nil { + return ret, fmt.Errorf("conversion failed to version %s for type %s - %w", outGVK.Version, formatGVK(inGVK), err) + } + + if outGVK.Version == conv.HubVersion { + out = hub + } else if err = hub.ConvertTo(ctx, out); err != nil { + return ret, fmt.Errorf("conversion failed to version %s for type %s - %w", outGVK.Version, formatGVK(inGVK), err) + } + + out.GetObjectKind().SetGroupVersionKind(outGVK) + + if defaultable, ok := out.(apis.Defaultable); ok { + defaultable.SetDefaults(ctx) + } + + if ret.Raw, err = json.Marshal(out); err != nil { + return ret, fmt.Errorf("unable to marshal output: %w", err) + } + return ret, nil +} + +func parseGVK(in runtime.RawExtension) (schema.GroupVersionKind, error) { + var ( + typeMeta metav1.TypeMeta + gvk schema.GroupVersionKind + ) + + if err := json.Unmarshal(in.Raw, &typeMeta); err != nil { + return gvk, fmt.Errorf("error parsing type meta %q - %w", string(in.Raw), err) + } + + gv, err := schema.ParseGroupVersion(typeMeta.APIVersion) + if err != nil { + return gvk, fmt.Errorf("error parsing GV %q: %w", typeMeta.APIVersion, err) + } + gvk = gv.WithKind(typeMeta.Kind) + + if gvk.Group == "" || gvk.Version == "" || gvk.Kind == "" { + return gvk, fmt.Errorf("invalid GroupVersionKind %v", gvk) + } + + return gvk, nil +} + +func parseAPIVersion(apiVersion string, kind string) (schema.GroupVersionKind, error) { + gv, err := schema.ParseGroupVersion(apiVersion) + if err != nil { + err = fmt.Errorf("desired API version %q is not valid", apiVersion) + return schema.GroupVersionKind{}, err + } + + if !isValidGV(gv) { + err = fmt.Errorf("desired API version %q is not valid", apiVersion) + return schema.GroupVersionKind{}, err + } + + return gv.WithKind(kind), nil +} + +func formatGVK(gvk schema.GroupVersionKind) string { + return fmt.Sprintf("[kind=%s group=%s version=%s]", gvk.Kind, gvk.Group, gvk.Version) +} + +func formatGK(gk schema.GroupKind) string { + return fmt.Sprintf("[kind=%s group=%s]", gk.Kind, gk.Group) +} + +func isValidGV(gk schema.GroupVersion) bool { + return gk.Group != "" && gk.Version != "" +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go new file mode 100644 index 000000000..362d12b3d --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/conversion_test.go @@ -0,0 +1,630 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + // injection + _ "knative.dev/pkg/client/injection/apiextensions/informers/apiextensions/v1/customresourcedefinition/fake" + _ "knative.dev/pkg/injection/clients/namespacedkube/informers/core/v1/secret/fake" + + "github.com/Azure/karpenter-provider-azure/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "knative.dev/pkg/webhook" + + . "knative.dev/pkg/reconciler/testing" +) + +var ( + webhookPath = "/convert" + testGK = schema.GroupKind{ + Group: internal.Group, + Kind: internal.Kind, + } + + zygotes = map[string]ConvertibleObject{ + "v1": &internal.V1Resource{}, + "v2": &internal.V2Resource{}, + "v3": &internal.V3Resource{}, + "error": &internal.ErrorResource{}, + } + + kinds = map[schema.GroupKind]GroupKindConversion{ + testGK: { + DefinitionName: "resource.webhook.pkg.knative.dev", + HubVersion: "v1", + Zygotes: zygotes, + }, + } + + rawOpt = cmp.Transformer("raw", func(res []runtime.RawExtension) []string { + result := make([]string, 0, len(res)) + for _, re := range res { + result = append(result, string(re.Raw)) + } + return result + }) + + cmpOpts = []cmp.Option{ + rawOpt, + } +) + +func testAPIVersion(version string) string { + return testGK.WithVersion(version).GroupVersion().String() +} + +func TestWebhookPath(t *testing.T) { + ctx, _ := SetupFakeContext(t) + ctx = webhook.WithOptions(ctx, webhook.Options{ + SecretName: "webhook-secret", + }) + + controller := NewConversionController(ctx, "/some-path", nil, nil) + conversion := controller.Reconciler.(webhook.ConversionController) + + if got, want := conversion.Path(), "/some-path"; got != want { + t.Errorf("expected controller to return provided path got: %q, want: %q", got, want) + } +} + +func TestConversionToHub(t *testing.T) { + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v1"), + Objects: []runtime.RawExtension{ + toRaw(t, internal.NewV2("bing")), + toRaw(t, internal.NewV3("bang")), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{Status: metav1.StatusSuccess}, + ConvertedObjects: []runtime.RawExtension{ + toRaw(t, internal.NewV1("bing")), + toRaw(t, internal.NewV1("bang")), + }, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } +} + +func TestConversionFromHub(t *testing.T) { + tests := []struct { + version string + in runtime.Object + out runtime.Object + }{{ + version: "v2", + in: internal.NewV1("bing"), + out: internal.NewV2("bing"), + }, { + version: "v3", + in: internal.NewV1("bing"), + out: internal.NewV3("bing"), + }} + + for _, test := range tests { + t.Run(test.version, func(t *testing.T) { + ctx, conversion := newConversion(t) + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion(test.version), + Objects: []runtime.RawExtension{ + toRaw(t, test.in), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{Status: metav1.StatusSuccess}, + ConvertedObjects: []runtime.RawExtension{ + toRaw(t, test.out), + }, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + }) + } +} + +func TestConversionThroughHub(t *testing.T) { + tests := []struct { + name string + version string + in runtime.Object + out runtime.Object + }{{ + name: "v3 to v2", + version: "v2", + in: internal.NewV3("bing"), + out: internal.NewV2("bing"), + }, { + name: "v2 to v3", + version: "v3", + in: internal.NewV2("bang"), + out: internal.NewV3("bang"), + }} + + for _, test := range tests { + t.Run(test.version, func(t *testing.T) { + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion(test.version), + Objects: []runtime.RawExtension{ + toRaw(t, test.in), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{Status: metav1.StatusSuccess}, + ConvertedObjects: []runtime.RawExtension{ + toRaw(t, test.out), + }, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + }) + } +} + +func TestConversionErrorBadGVK(t *testing.T) { + tests := []struct { + name string + gvk schema.GroupVersionKind + }{{ + name: "empty group", + gvk: schema.GroupVersionKind{ + Version: "v1", + Kind: "Resource", + }, + }, { + name: "empty version", + gvk: schema.GroupVersionKind{ + Group: "webhook.pkg.knative.dev", + Kind: "Resource", + }, + }, { + name: "empty kind", + gvk: schema.GroupVersionKind{ + Group: "webhook.pkg.knative.dev", + Version: "v1", + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + obj := internal.NewV2("bing") + obj.SetGroupVersionKind(test.gvk) + + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v1"), + Objects: []runtime.RawExtension{ + toRaw(t, obj), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + cmpopts.EquateEmpty(), + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "invalid GroupVersionKind") { + t.Errorf("expected message to start with 'invalid GroupVersionKind' got %q", got.Result.Message) + } + }) + } +} + +func TestConversionUnknownInputGVK(t *testing.T) { + unknownObj := &unstructured.Unstructured{} + unknownObj.SetGroupVersionKind(schema.GroupVersionKind{ + Group: "some.api.group.dev", + Version: "v1", + Kind: "Resource", + }) + + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v3"), + Objects: []runtime.RawExtension{ + toRaw(t, unknownObj), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Message: "no conversion support for type [kind=Resource group=some.api.group.dev]", + Status: metav1.StatusFailure, + }, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } +} + +func TestConversionInvalidTypeMeta(t *testing.T) { + ctx, conversion := newConversionWithKinds(t, nil) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: "some-version", + Objects: []runtime.RawExtension{ + {Raw: []byte("}")}, + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + cmpopts.EquateEmpty(), + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "error parsing type meta") { + t.Errorf("expected message to start with 'error parsing type meta' got %q", got.Result.Message) + } +} + +func TestConversionFailureToUnmarshalInput(t *testing.T) { + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v1"), + Objects: []runtime.RawExtension{ + toRaw(t, internal.NewErrorResource(internal.ErrorUnmarshal)), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + cmpopts.EquateEmpty(), + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "unable to unmarshal input") { + t.Errorf("expected message to start with 'unable to unmarshal input' got %q", got.Result.Message) + } +} + +func TestConversionFailureToMarshalOutput(t *testing.T) { + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("error"), + Objects: []runtime.RawExtension{ + // This property should make the Marshal on the + // ErrorResource to fail + toRaw(t, internal.NewV1(internal.ErrorMarshal)), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + cmpopts.EquateEmpty(), + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "unable to marshal output") { + t.Errorf("expected message to start with 'unable to marshal output' got %q", got.Result.Message) + } +} + +func TestConversionFailureToConvert(t *testing.T) { + // v1 => error resource => v3 + kinds := map[schema.GroupKind]GroupKindConversion{ + testGK: { + DefinitionName: "resource.webhook.pkg.knative.dev", + HubVersion: "error", + Zygotes: zygotes, + }, + } + + tests := []struct { + name string + errorOn string + }{{ + name: "error converting from", + errorOn: internal.ErrorConvertFrom, + }, { + name: "error converting to", + errorOn: internal.ErrorConvertTo, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, conversion := newConversionWithKinds(t, kinds) + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v3"), + Objects: []runtime.RawExtension{ + // Insert failure here + toRaw(t, internal.NewV1(test.errorOn)), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + rawOpt, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "conversion failed") { + t.Errorf("expected message to start with 'conversion failed' got %q", got.Result.Message) + } + }) + } +} + +func TestConversionFailureInvalidDesiredAPIVersion(t *testing.T) { + tests := []struct { + name string + version string + }{{ + name: "multiple path segments", + version: "bad-api-version/v1/v2", + }, { + name: "empty", + version: "", + }, { + name: "path segment", + version: "/", + }, { + name: "no version", + version: "some.api.group", + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, conversion := newConversion(t) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: test.version, + Objects: []runtime.RawExtension{ + toRaw(t, internal.NewV1("bing")), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Message: fmt.Sprintf("desired API version %q is not valid", test.version), + Status: metav1.StatusFailure, + }, + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + }) + } +} + +func TestConversionMissingZygotes(t *testing.T) { + // Assume we're converting from + // v2 (input) => v1 (hub) => v3 (output) + tests := []struct { + name string + zygotes map[string]ConvertibleObject + }{{ + name: "missing input", + zygotes: map[string]ConvertibleObject{ + "v1": &internal.V1Resource{}, + "v3": &internal.V3Resource{}, + }, + }, { + name: "missing output", + zygotes: map[string]ConvertibleObject{ + "v1": &internal.V1Resource{}, + "v2": &internal.V2Resource{}, + }, + }, { + name: "missing hub", + zygotes: map[string]ConvertibleObject{ + "v2": &internal.V2Resource{}, + "v3": &internal.V3Resource{}, + }, + }} + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + kinds = map[schema.GroupKind]GroupKindConversion{ + testGK: { + DefinitionName: "resource.webhook.pkg.knative.dev", + HubVersion: "v1", + Zygotes: test.zygotes, + }, + } + + ctx, conversion := newConversionWithKinds(t, kinds) + + req := &apixv1.ConversionRequest{ + UID: "some-uid", + DesiredAPIVersion: testAPIVersion("v3"), + Objects: []runtime.RawExtension{ + toRaw(t, internal.NewV2("bing")), + }, + } + + want := &apixv1.ConversionResponse{ + UID: "some-uid", + Result: metav1.Status{ + Status: metav1.StatusFailure, + }, + } + + cmpOpts := []cmp.Option{ + cmpopts.IgnoreFields(metav1.Status{}, "Message"), + cmpopts.EquateEmpty(), + } + + got := conversion.Convert(ctx, req) + if diff := cmp.Diff(want, got, cmpOpts...); diff != "" { + t.Error("unexpected response:", diff) + } + + if !strings.HasPrefix(got.Result.Message, "conversion not supported") { + t.Errorf("expected message to start with 'conversion not supported' got %q", got.Result.Message) + } + }) + } +} + +func TestContextDecoration(t *testing.T) { + ctx, _ := SetupFakeContext(t) + ctx = webhook.WithOptions(ctx, webhook.Options{ + SecretName: "webhook-secret", + }) + + decoratorCalled := false + decorator := func(ctx context.Context) context.Context { + decoratorCalled = true + return ctx + } + + controller := NewConversionController(ctx, webhookPath, kinds, decorator) + r := controller.Reconciler.(*reconciler) + r.Convert(ctx, &apixv1.ConversionRequest{}) + + if !decoratorCalled { + t.Errorf("context decorator was not invoked") + } +} + +func toRaw(t *testing.T, obj runtime.Object) runtime.RawExtension { + t.Helper() + + raw, err := json.Marshal(obj) + if err != nil { + t.Fatal("unable to marshal resource:", err) + } + + return runtime.RawExtension{Raw: raw} +} + +func newConversion(t *testing.T) (context.Context, webhook.ConversionController) { + return newConversionWithKinds(t, kinds) +} + +func newConversionWithKinds( + t *testing.T, + kinds map[schema.GroupKind]GroupKindConversion, +) ( + context.Context, + webhook.ConversionController, +) { + ctx, _ := SetupFakeContext(t) + ctx = webhook.WithOptions(ctx, webhook.Options{ + SecretName: "webhook-secret", + }) + + controller := NewConversionController(ctx, webhookPath, kinds, nil) + return ctx, controller.Reconciler.(*reconciler) +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go new file mode 100644 index 000000000..14c56acd1 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/types.go @@ -0,0 +1,278 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" +) + +const ( + // Group specifies the group of the test resource + Group = "webhook.pkg.knative.dev" + + // Kind specifies the kind of the test resource + Kind = "Resource" + + // ErrorMarshal when assigned to the Spec.Property of the ErrorResource + // will cause json marshaling of the resource to fail + ErrorMarshal = "marshal" + + // ErrorUnmarshal when assigned to the Spec.Property of the ErrorResource + // will cause json unmarshalling of the resource to fail + ErrorUnmarshal = "unmarshal" + + // ErrorConvertTo when assigned to the Spec.Property of the ErrorResource + // will cause ConvertTo to fail + ErrorConvertTo = "convertTo" + + // ErrorConvertFrom when assigned to the Spec.Property of the ErrorResource + // will cause ConvertFrom to fail + ErrorConvertFrom = "convertFrom" +) + +type ( + // V1Resource will never has a prefix or suffix on Spec.Property + // This type is used for testing conversion webhooks + // + // +k8s:deepcopy-gen=true + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + V1Resource struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec Spec `json:"spec"` + } + + // V2Resource will always have a 'prefix/' in front of it's property + // This type is used for testing conversion webhooks + // + // +k8s:deepcopy-gen=true + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + V2Resource struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec Spec `json:"spec"` + } + + // V3Resource will always have a '/suffix' in front of it's property + // This type is used for testing conversion webhooks + // + // +k8s:deepcopy-gen=true + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + V3Resource struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec SpecWithDefault `json:"spec"` + } + + // ErrorResource explodes in various settings depending on the property + // set. Use the Error* constants + // + // This type is used for testing conversion webhooks + // + // +k8s:deepcopy-gen=true + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + ErrorResource struct { + // We embed the V1Resource as an easy way to still marshal & unmarshal + // this type without infinite loops - since we override the methods + // in order to induce failures + V1Resource `json:",inline"` + } + + // Spec holds our fancy string property + Spec struct { + Property string `json:"prop"` + } + + // SpecWithDefault holds two fancy string properties + SpecWithDefault struct { + Property string `json:"prop"` + NewProperty string `json:"defaulted_prop"` + } +) + +var ( + _ apis.Convertible = (*V1Resource)(nil) + _ apis.Convertible = (*V2Resource)(nil) + _ apis.Convertible = (*V3Resource)(nil) + _ apis.Convertible = (*ErrorResource)(nil) + + _ apis.Defaultable = (*V3Resource)(nil) +) + +// NewV1 returns a V1Resource with Spec.Property set +// to prop +func NewV1(prop string) *V1Resource { + return &V1Resource{ + TypeMeta: metav1.TypeMeta{ + Kind: Kind, + APIVersion: Group + "/v1", + }, + Spec: Spec{ + Property: prop, + }, + } +} + +// NewV2 returns a V2Resource with Spec.Property set +// to 'prefix/' + prop +func NewV2(prop string) *V2Resource { + return &V2Resource{ + TypeMeta: metav1.TypeMeta{ + Kind: Kind, + APIVersion: Group + "/v2", + }, + Spec: Spec{ + Property: "prefix/" + prop, + }, + } +} + +// NewV3 returns a V3Resource with Spec.Property set +// to prop + '/suffix' +func NewV3(prop string) *V3Resource { + v3 := &V3Resource{ + TypeMeta: metav1.TypeMeta{ + Kind: Kind, + APIVersion: Group + "/v3", + }, + Spec: SpecWithDefault{ + Property: prop + "/suffix", + }, + } + v3.SetDefaults(context.Background()) + return v3 +} + +// NewErrorResource returns an ErrorResource with Spec.Property set +// to failure +func NewErrorResource(failure string) *ErrorResource { + return &ErrorResource{ + V1Resource: V1Resource{ + TypeMeta: metav1.TypeMeta{ + Kind: Kind, + APIVersion: Group + "/error", + }, + Spec: Spec{ + Property: failure, + }, + }, + } +} + +// ConvertTo implements apis.Convertible +func (r *V1Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { + switch sink := to.(type) { + case *V2Resource: + sink.Spec.Property = "prefix/" + r.Spec.Property + case *V3Resource: + sink.Spec.Property = r.Spec.Property + "/suffix" + case *ErrorResource: + sink.Spec.Property = r.Spec.Property + case *V1Resource: + sink.Spec.Property = r.Spec.Property + default: + return fmt.Errorf("unsupported type %T", sink) + } + return nil +} + +// ConvertFrom implements apis.Convertible +func (r *V1Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { + switch source := from.(type) { + case *V2Resource: + r.Spec.Property = strings.TrimPrefix(source.Spec.Property, "prefix/") + case *V3Resource: + r.Spec.Property = strings.TrimSuffix(source.Spec.Property, "/suffix") + case *ErrorResource: + r.Spec.Property = source.Spec.Property + case *V1Resource: + r.Spec.Property = source.Spec.Property + default: + return fmt.Errorf("unsupported type %T", source) + } + return nil +} + +// SetDefaults implements apis.Defaultable +func (r *V3Resource) SetDefaults(ctx context.Context) { + if r.Spec.NewProperty == "" { + r.Spec.NewProperty = "defaulted" + } +} + +// ConvertTo implements apis.Convertible +func (*V2Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { + panic("unimplemented") +} + +// ConvertFrom implements apis.Convertible +func (*V2Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { + panic("unimplemented") +} + +// ConvertTo implements apis.Convertible +func (*V3Resource) ConvertTo(ctx context.Context, to apis.Convertible) error { + panic("unimplemented") +} + +// ConvertFrom implements apis.Convertible +func (*V3Resource) ConvertFrom(ctx context.Context, from apis.Convertible) error { + panic("unimplemented") +} + +// ConvertTo implements apis.Convertible +func (e *ErrorResource) ConvertTo(ctx context.Context, to apis.Convertible) error { + if e.Spec.Property == ErrorConvertTo { + return errors.New("boooom - convert up") + } + + return e.V1Resource.ConvertTo(ctx, to) +} + +// ConvertFrom implements apis.Convertible +func (e *ErrorResource) ConvertFrom(ctx context.Context, from apis.Convertible) error { + err := e.V1Resource.ConvertFrom(ctx, from) + + if err == nil && e.Spec.Property == ErrorConvertFrom { + err = errors.New("boooom - convert down") + } + return err +} + +// UnmarshalJSON implements json.Unmarshaler +func (e *ErrorResource) UnmarshalJSON(data []byte) (err error) { + err = json.Unmarshal(data, &e.V1Resource) + if err == nil && e.Spec.Property == ErrorUnmarshal { + err = errors.New("boooom - unmarshal json") + } + return +} + +// MarshalJSON implements json.Marshaler +func (e *ErrorResource) MarshalJSON() ([]byte, error) { + if e.Spec.Property == ErrorMarshal { + return nil, errors.New("boooom - marshal json") + } + return json.Marshal(e.V1Resource) +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go new file mode 100644 index 000000000..6ea895a85 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/internal/zz_generated.deepcopy.go @@ -0,0 +1,132 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright 2022 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package internal + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ErrorResource) DeepCopyInto(out *ErrorResource) { + *out = *in + in.V1Resource.DeepCopyInto(&out.V1Resource) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ErrorResource. +func (in *ErrorResource) DeepCopy() *ErrorResource { + if in == nil { + return nil + } + out := new(ErrorResource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ErrorResource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *V1Resource) DeepCopyInto(out *V1Resource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V1Resource. +func (in *V1Resource) DeepCopy() *V1Resource { + if in == nil { + return nil + } + out := new(V1Resource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *V1Resource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *V2Resource) DeepCopyInto(out *V2Resource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V2Resource. +func (in *V2Resource) DeepCopy() *V2Resource { + if in == nil { + return nil + } + out := new(V2Resource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *V2Resource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *V3Resource) DeepCopyInto(out *V3Resource) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new V3Resource. +func (in *V3Resource) DeepCopy() *V3Resource { + if in == nil { + return nil + } + out := new(V3Resource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *V3Resource) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go new file mode 100644 index 000000000..488d54987 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options.go @@ -0,0 +1,46 @@ +// Portions Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package conversion + +import ( + "context" + + "k8s.io/apimachinery/pkg/runtime/schema" +) + +type options struct { + path string + wc func(context.Context) context.Context + kinds map[schema.GroupKind]GroupKindConversion +} + +type OptionFunc func(*options) + +func WithKinds(kinds map[schema.GroupKind]GroupKindConversion) OptionFunc { + return func(o *options) { + o.kinds = kinds + } +} + +func WithPath(path string) OptionFunc { + return func(o *options) { + o.path = path + } +} + +func WithWrapContext(f func(context.Context) context.Context) OptionFunc { + return func(o *options) { + o.wc = f + } +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go new file mode 100644 index 000000000..438fffd91 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/options_test.go @@ -0,0 +1,38 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "reflect" + "testing" +) + +func TestOptions(t *testing.T) { + got := &options{} + WithPath("path")(got) + + want := &options{ + path: "path", + // we can't compare wc as functions are not + // comparable in golang (thus it needs to be + // done indirectly) + } + + if !reflect.DeepEqual(got, want) { + t.Error("option was not applied") + } +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go new file mode 100644 index 000000000..ed0fadc28 --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/reconciler.go @@ -0,0 +1,123 @@ +// Portions Copyright (c) Microsoft Corporation. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package conversion + +import ( + "context" + "fmt" + + "go.uber.org/zap" + apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apixclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + apixlisters "k8s.io/apiextensions-apiserver/pkg/client/listers/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + corelisters "k8s.io/client-go/listers/core/v1" + "knative.dev/pkg/controller" + "knative.dev/pkg/kmp" + "knative.dev/pkg/logging" + "knative.dev/pkg/ptr" + pkgreconciler "knative.dev/pkg/reconciler" + "knative.dev/pkg/system" + "knative.dev/pkg/webhook" + certresources "knative.dev/pkg/webhook/certificates/resources" +) + +type reconciler struct { + pkgreconciler.LeaderAwareFuncs + + kinds map[schema.GroupKind]GroupKindConversion + path string + secretName string + withContext func(context.Context) context.Context + + secretLister corelisters.SecretLister + crdLister apixlisters.CustomResourceDefinitionLister + client apixclient.Interface +} + +var ( + _ webhook.ConversionController = (*reconciler)(nil) + _ controller.Reconciler = (*reconciler)(nil) + _ pkgreconciler.LeaderAware = (*reconciler)(nil) +) + +// Path implements webhook.ConversionController +func (r *reconciler) Path() string { + return r.path +} + +// Reconciler implements controller.Reconciler +func (r *reconciler) Reconcile(ctx context.Context, key string) error { + logger := logging.FromContext(ctx) + + if !r.IsLeaderFor(types.NamespacedName{Name: key}) { + return controller.NewSkipKey(key) + } + + // Look up the webhook secret, and fetch the CA cert bundle. + secret, err := r.secretLister.Secrets(system.Namespace()).Get(r.secretName) + if err != nil { + logger.Errorw("Error fetching secret", zap.Error(err)) + return err + } + + cacert, ok := secret.Data[certresources.CACert] + if !ok { + return fmt.Errorf("secret %q is missing %q key", r.secretName, certresources.CACert) + } + + return r.reconcileCRD(ctx, cacert, key) +} + +func (r *reconciler) reconcileCRD(ctx context.Context, cacert []byte, key string) error { + logger := logging.FromContext(ctx) + + configuredCRD, err := r.crdLister.Get(key) + if err != nil { + return fmt.Errorf("error retrieving crd: %w", err) + } + + crd := configuredCRD.DeepCopy() + + if crd.Spec.Conversion == nil || + crd.Spec.Conversion.Strategy != apixv1.WebhookConverter || + crd.Spec.Conversion.Webhook.ClientConfig == nil || + (crd.Spec.Conversion.Webhook.ClientConfig.Service == nil && crd.Spec.Conversion.Webhook.ClientConfig.URL == nil) { // AKS customized + return fmt.Errorf("custom resource %q isn't configured for webhook conversion", key) + } + + crd.Spec.Conversion.Webhook.ClientConfig.CABundle = cacert + + // AKS customized + if crd.Spec.Conversion.Webhook.ClientConfig.Service != nil { // This line is changed from upstream + crd.Spec.Conversion.Webhook.ClientConfig.Service.Path = ptr.String(r.path) + } + // end customization + + if ok, err := kmp.SafeEqual(configuredCRD, crd); err != nil { + return fmt.Errorf("error diffing custom resource definitions: %w", err) + } else if !ok { + logger.Infof("updating CRD") + crdClient := r.client.ApiextensionsV1().CustomResourceDefinitions() + if _, err := crdClient.Update(ctx, crd, metav1.UpdateOptions{}); err != nil { + return fmt.Errorf("failed to update webhook: %w", err) + } + } else { + logger.Info("CRD is up to date") + } + + return nil +} diff --git a/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go new file mode 100644 index 000000000..dc812302c --- /dev/null +++ b/pkg/alt/knative/pkg/webhook/resourcesemantics/conversion/table_test.go @@ -0,0 +1,290 @@ +/* +Portions Copyright (c) Microsoft Corporation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conversion + +import ( + "context" + "testing" + + corev1 "k8s.io/api/core/v1" + apixv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgotesting "k8s.io/client-go/testing" + apixclient "knative.dev/pkg/client/injection/apiextensions/client/fake" + "knative.dev/pkg/configmap" + "knative.dev/pkg/controller" + "knative.dev/pkg/ptr" + "knative.dev/pkg/system" + certresources "knative.dev/pkg/webhook/certificates/resources" + + . "knative.dev/pkg/reconciler/testing" + . "knative.dev/pkg/webhook/testing" +) + +func TestReconcile(t *testing.T) { + key := "some.crd.group.dev" + path := "/some/path" + secretName := "webhook-secret" + + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: system.Namespace(), + }, + Data: map[string][]byte{ + certresources.ServerKey: []byte("present"), + certresources.ServerCert: []byte("present"), + certresources.CACert: []byte("present"), + }, + } + + table := TableTest{{ + Name: "no secret", + Key: key, + WantErr: true, + }, { + Name: "secret missing CA Cert", + Key: key, + Objects: []runtime.Object{&corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: system.Namespace(), + }, + Data: map[string][]byte{ + certresources.ServerKey: []byte("present"), + certresources.ServerCert: []byte("present"), + // certresources.CACert: []byte("missing"), + }, + }}, + WantErr: true, + }, { + Name: "secret exists, but CRD does not", + Key: key, + Objects: []runtime.Object{secret}, + WantErr: true, + }, { + Name: "secret and CRD exist, missing service reference", + Key: key, + Objects: []runtime.Object{ + secret, + &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{}, + }, + }, + }, + WantErr: true, + }, { + Name: "secret and CRD exist, missing other stuff", + Key: key, + Objects: []runtime.Object{ + secret, + &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + }, + }, + }, + }, + }, + }, + }, + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + // Path is added. + Path: ptr.String(path), + }, + // CABundle is added. + CABundle: []byte("present"), + }, + }, + }, + }, + }, + }}, + }, { + Name: "secret and CRD exist, incorrect fields", + Key: key, + Objects: []runtime.Object{ + secret, + &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + // Incorrect path + Path: ptr.String("/incorrect"), + }, + // CABundle is added. + CABundle: []byte("incorrect"), + }, + }, + }, + }, + }, + }, + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + // Path is added. + Path: ptr.String(path), + }, + // CABundle is added. + CABundle: []byte("present"), + }, + }, + }, + }, + }, + }}, + }, { + Name: "failed to update custom resource definition", + Key: key, + WantErr: true, + WithReactors: []clientgotesting.ReactionFunc{ + InduceFailure("update", "customresourcedefinitions"), + }, + Objects: []runtime.Object{ + secret, + &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + // Incorrect path + Path: ptr.String("/incorrect"), + }, + // CABundle is added. + CABundle: []byte("incorrect"), + }, + }, + }, + }, + }, + }, + WantUpdates: []clientgotesting.UpdateActionImpl{{ + Object: &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + // Path is added. + Path: ptr.String(path), + }, + // CABundle is added. + CABundle: []byte("present"), + }, + }, + }, + }, + }, + }}, + }, { + Name: "stable", + Key: key, + Objects: []runtime.Object{ + secret, + &apixv1.CustomResourceDefinition{ + ObjectMeta: metav1.ObjectMeta{ + Name: key, + }, + Spec: apixv1.CustomResourceDefinitionSpec{ + Conversion: &apixv1.CustomResourceConversion{ + Strategy: apixv1.WebhookConverter, + Webhook: &apixv1.WebhookConversion{ + ClientConfig: &apixv1.WebhookClientConfig{ + Service: &apixv1.ServiceReference{ + Namespace: system.Namespace(), + Name: "webhook", + Path: ptr.String(path), + }, + // CABundle is added. + CABundle: []byte("present"), + }, + }, + }, + }, + }, + }, + }} + + table.Test(t, MakeFactory(func(ctx context.Context, listers *Listers, cmw configmap.Watcher) controller.Reconciler { + return &reconciler{ + kinds: kinds, + path: path, + secretName: secretName, + secretLister: listers.GetSecretLister(), + crdLister: listers.GetCustomResourceDefinitionLister(), + client: apixclient.Get(ctx), + } + })) +} diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 7d82a4083..4c670e038 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -20,21 +20,33 @@ import ( "context" "encoding/base64" "fmt" + "sync" + "github.com/awslabs/operatorpkg/controller" + "github.com/awslabs/operatorpkg/object" + "github.com/awslabs/operatorpkg/status" "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" "k8s.io/client-go/transport" + knativeinjection "knative.dev/pkg/injection" "knative.dev/pkg/ptr" + "sigs.k8s.io/controller-runtime/pkg/log" karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/operator/injection" + karpenteroptions "sigs.k8s.io/karpenter/pkg/operator/options" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + webhooksalt "github.com/Azure/karpenter-provider-azure/pkg/alt/karpenter-core/pkg/webhooks" "github.com/Azure/karpenter-provider-azure/pkg/auth" azurecache "github.com/Azure/karpenter-provider-azure/pkg/cache" + "github.com/Azure/karpenter-provider-azure/pkg/operator/options" "github.com/Azure/karpenter-provider-azure/pkg/providers/imagefamily" "github.com/Azure/karpenter-provider-azure/pkg/providers/instance" @@ -64,6 +76,9 @@ type Operator struct { InstanceTypesProvider instancetype.Provider InstanceProvider *instance.DefaultProvider LoadBalancerProvider *loadbalancer.Provider + + // Copied from the core Operator because we control our own webhooks + webhooks []knativeinjection.ControllerConstructor } func NewOperator(ctx context.Context, operator *operator.Operator) (context.Context, *Operator) { @@ -143,6 +158,50 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont } } +// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks +func (o *Operator) WithControllers(ctx context.Context, controllers ...controller.Controller) *Operator { + for _, c := range controllers { + lo.Must0(c.Register(ctx, o.Manager)) + } + return o +} + +// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks +func (o *Operator) WithWebhooks(ctx context.Context, ctors ...knativeinjection.ControllerConstructor) *Operator { + if !karpenteroptions.FromContext(ctx).DisableWebhook { + o.webhooks = append(o.webhooks, ctors...) + lo.Must0(o.Manager.AddReadyzCheck("webhooks", webhooksalt.HealthProbe(ctx))) + lo.Must0(o.Manager.AddHealthzCheck("webhooks", webhooksalt.HealthProbe(ctx))) + } + return o +} + +// Copied from karpenter-core pkg/operator/operator.go, needed for webhooks +func (o *Operator) Start(ctx context.Context, cp cloudprovider.CloudProvider) { + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + lo.Must0(o.Manager.Start(ctx)) + }() + if karpenteroptions.FromContext(ctx).DisableWebhook { + log.FromContext(ctx).Info("conversion webhooks are disabled") + } else { + wg.Add(1) + go func() { + defer wg.Done() + // Taking the first supported NodeClass to be the default NodeClass + gvk := lo.Map(cp.GetSupportedNodeClasses(), func(nc status.Object, _ int) schema.GroupVersionKind { + return object.GVK(nc) + }) + ctx = injection.WithNodeClasses(ctx, gvk) + ctx = injection.WithClient(ctx, o.GetClient()) + webhooksalt.Start(ctx, o.GetConfig(), o.webhooks...) // This is our alt copy of webhooks that can support multiple apiservers + }() + } + wg.Wait() +} + func GetAZConfig() (*auth.Config, error) { cfg, err := auth.BuildAzureConfig() if err != nil { From 20ccbfdfc0e12b4363e75ecc63f3442e63beec21 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 18:52:57 +0000 Subject: [PATCH 41/47] chore: remove failSwapOn from kubelet settings in AKSNodeClass --- .../templates/karpenter.azure.com_aksnodeclasses.yaml | 6 ------ pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml | 6 ------ pkg/apis/v1alpha2/aksnodeclass.go | 5 ----- pkg/apis/v1alpha2/zz_generated.deepcopy.go | 5 ----- 4 files changed, 22 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index 7153a6e23..e3c7878d9 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -105,12 +105,6 @@ spec: - none - static type: string - failSwapOn: - description: |- - failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. - Default: true - kubebuilder:default:=true - type: boolean imageGCHighThresholdPercent: description: |- ImageGCHighThresholdPercent is the percent of disk usage after which image diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index 7153a6e23..e3c7878d9 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -105,12 +105,6 @@ spec: - none - static type: string - failSwapOn: - description: |- - failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. - Default: true - kubebuilder:default:=true - type: boolean imageGCHighThresholdPercent: description: |- ImageGCHighThresholdPercent is the percent of disk usage after which image diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index eb68dfb4f..4c06bb6a2 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -123,11 +123,6 @@ type KubeletConfiguration struct { // TODO: validation // +optional AllowedUnsafeSysctls []string `json:"allowedUnsafeSysctls,omitempty"` - // failSwapOn tells the Kubelet to fail to start if swap is enabled on the node. - // Default: true - // kubebuilder:default:=true - // +optional - FailSwapOn *bool `json:"failSwapOn,omitempty"` // containerLogMaxSize is a quantity defining the maximum size of the container log // file before it is rotated. For example: "5Mi" or "256Ki". // Default: "10Mi" diff --git a/pkg/apis/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/v1alpha2/zz_generated.deepcopy.go index a96bf6c4e..8646043cf 100644 --- a/pkg/apis/v1alpha2/zz_generated.deepcopy.go +++ b/pkg/apis/v1alpha2/zz_generated.deepcopy.go @@ -205,11 +205,6 @@ func (in *KubeletConfiguration) DeepCopyInto(out *KubeletConfiguration) { *out = make([]string, len(*in)) copy(*out, *in) } - if in.FailSwapOn != nil { - in, out := &in.FailSwapOn, &out.FailSwapOn - *out = new(bool) - **out = **in - } if in.ContainerLogMaxFiles != nil { in, out := &in.ContainerLogMaxFiles, &out.ContainerLogMaxFiles *out = new(int32) From 27fb940c2f2c2642a06d2c4412f33904ede2cc2d Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 19:25:08 +0000 Subject: [PATCH 42/47] fix: populate nodeClaim.Status.ImageID --- pkg/cloudprovider/cloudprovider.go | 4 +++- pkg/utils/utils.go | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index f164b1c98..8a27aa9b1 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -367,7 +367,9 @@ func (c *CloudProvider) instanceToNodeClaim(ctx context.Context, vm *armcompute. nodeClaim.DeletionTimestamp = &metav1.Time{Time: time.Now()} } nodeClaim.Status.ProviderID = utils.ResourceIDToProviderID(ctx, *vm.ID) - //TOFIX (could be nil, at least when faked) nodeClaim.Status.ImageID = utils.ImageReferenceToString(*vm.Properties.StorageProfile.ImageReference) + if vm.Properties != nil && vm.Properties.StorageProfile != nil && vm.Properties.StorageProfile.ImageReference != nil { + nodeClaim.Status.ImageID = utils.ImageReferenceToString(vm.Properties.StorageProfile.ImageReference) + } return nodeClaim, nil } diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index f1a1ff692..55750f6f2 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -78,7 +78,7 @@ func WithDefaultFloat64(key string, def float64) float64 { return f } -func ImageReferenceToString(imageRef armcompute.ImageReference) string { +func ImageReferenceToString(imageRef *armcompute.ImageReference) string { // Check for Custom Image if imageRef.ID != nil && *imageRef.ID != "" { return *imageRef.ID From 07052f3cbdcd6944ee5d9dde8e9b7b089aa3ffec Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 20:20:27 +0000 Subject: [PATCH 43/47] fix: record NodeClass hash and add drift on static fields --- pkg/cloudprovider/cloudprovider.go | 44 ++++++++++++---------------- pkg/cloudprovider/drift.go | 46 ++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index 8a27aa9b1..b9c9764b3 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -125,10 +125,11 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *karpv1.NodeClaim) instanceType, _ := lo.Find(instanceTypes, func(i *cloudprovider.InstanceType) bool { return i.Name == string(lo.FromPtr(instance.Properties.HardwareProfile.VMSize)) }) - nc, err := c.instanceToNodeClaim(ctx, instance, instanceType) - - // TODO: record nodeclass hash & version + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1alpha2.AnnotationAKSNodeClassHash: nodeClass.Hash(), + v1alpha2.AnnotationAKSNodeClassHashVersion: v1alpha2.AKSNodeClassHashVersion, + }) return nc, err } @@ -204,39 +205,30 @@ func (c *CloudProvider) Delete(ctx context.Context, nodeClaim *karpv1.NodeClaim) } func (c *CloudProvider) IsDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim) (cloudprovider.DriftReason, error) { - if nodeClaim.Spec.NodeClassRef == nil { + // Not needed when GetInstanceTypes removes nodepool dependency + nodePoolName, ok := nodeClaim.Labels[karpv1.NodePoolLabelKey] + if !ok { return "", nil } - nodeClass, err := c.resolveNodeClassFromNodeClaim(ctx, nodeClaim) + nodePool := &karpv1.NodePool{} + if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePoolName}, nodePool); err != nil { + return "", client.IgnoreNotFound(err) + } + if nodePool.Spec.Template.Spec.NodeClassRef == nil { + return "", nil + } + nodeClass, err := c.resolveNodeClassFromNodePool(ctx, nodePool) if err != nil { if errors.IsNotFound(err) { - c.recorder.Publish(cloudproviderevents.NodeClaimFailedToResolveNodeClass(nodeClaim)) + c.recorder.Publish(cloudproviderevents.NodePoolFailedToResolveNodeClass(nodePool)) } return "", client.IgnoreNotFound(fmt.Errorf("resolving node class, %w", err)) } - - k8sVersionDrifted, err := c.isK8sVersionDrifted(ctx, nodeClaim) - if err != nil { - return "", err - } - if k8sVersionDrifted != "" { - return k8sVersionDrifted, nil - } - imageVersionDrifted, err := c.isImageVersionDrifted(ctx, nodeClaim) + driftReason, err := c.isNodeClassDrifted(ctx, nodeClaim, nodeClass) if err != nil { return "", err } - if imageVersionDrifted != "" { - return imageVersionDrifted, nil - } - subnetDrifted, err := c.isSubnetDrifted(ctx, nodeClaim, nodeClass) - if err != nil { - return "", err - } - if subnetDrifted != "" { - return subnetDrifted, nil - } - return "", nil + return driftReason, nil } // Name returns the CloudProvider implementation name. diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 545f4cc86..05b68127b 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -40,11 +40,57 @@ import ( ) const ( + NodeClassDrift cloudprovider.DriftReason = "NodeClassDrift" K8sVersionDrift cloudprovider.DriftReason = "K8sVersionDrift" ImageVersionDrift cloudprovider.DriftReason = "ImageVersionDrift" SubnetDrift cloudprovider.DriftReason = "SubnetDrift" ) +func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { + // First check if the node class is statically drifted to save on API calls. + if drifted := c.areStaticFieldsDrifted(nodeClaim, nodeClass); drifted != "" { + return drifted, nil + } + k8sVersionDrifted, err := c.isK8sVersionDrifted(ctx, nodeClaim) + if err != nil { + return "", err + } + if k8sVersionDrifted != "" { + return k8sVersionDrifted, nil + } + imageVersionDrifted, err := c.isImageVersionDrifted(ctx, nodeClaim) + if err != nil { + return "", err + } + if imageVersionDrifted != "" { + return imageVersionDrifted, nil + } + subnetDrifted, err := c.isSubnetDrifted(ctx, nodeClaim, nodeClass) + if err != nil { + return "", err + } + if subnetDrifted != "" { + return subnetDrifted, nil + } + return "", nil +} + +func (c *CloudProvider) areStaticFieldsDrifted(nodeClaim *karpv1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) cloudprovider.DriftReason { + nodeClassHash, foundNodeClassHash := nodeClass.Annotations[v1alpha2.AnnotationAKSNodeClassHash] + nodeClassHashVersion, foundNodeClassHashVersion := nodeClass.Annotations[v1alpha2.AnnotationAKSNodeClassHashVersion] + nodeClaimHash, foundNodeClaimHash := nodeClaim.Annotations[v1alpha2.AnnotationAKSNodeClassHash] + nodeClaimHashVersion, foundNodeClaimHashVersion := nodeClaim.Annotations[v1alpha2.AnnotationAKSNodeClassHashVersion] + + if !foundNodeClassHash || !foundNodeClaimHash || !foundNodeClassHashVersion || !foundNodeClaimHashVersion { + return "" + } + // validate that the hash version for the AKSNodeClass is the same as the NodeClaim before evaluating for static drift + if nodeClassHashVersion != nodeClaimHashVersion { + return "" + } + return lo.Ternary(nodeClassHash != nodeClaimHash, NodeClassDrift, "") +} + func (c *CloudProvider) isK8sVersionDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim) (cloudprovider.DriftReason, error) { logger := logging.FromContext(ctx) From 74fdd8a0c188999b8b7211a32aa10b38a58258ac Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:22:15 +0000 Subject: [PATCH 44/47] chore: rename variabled --- pkg/cloudprovider/drift.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index 05b68127b..98c1114bd 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -47,9 +47,9 @@ const ( ) func (c *CloudProvider) isNodeClassDrifted(ctx context.Context, nodeClaim *karpv1.NodeClaim, nodeClass *v1alpha2.AKSNodeClass) (cloudprovider.DriftReason, error) { - // First check if the node class is statically drifted to save on API calls. - if drifted := c.areStaticFieldsDrifted(nodeClaim, nodeClass); drifted != "" { - return drifted, nil + // First check if the node class is statically staticFieldsDrifted to save on API calls. + if staticFieldsDrifted := c.areStaticFieldsDrifted(nodeClaim, nodeClass); staticFieldsDrifted != "" { + return staticFieldsDrifted, nil } k8sVersionDrifted, err := c.isK8sVersionDrifted(ctx, nodeClaim) if err != nil { From 82e34f92cc6adddb55fe7d94d3aec62d63eef418 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:44:20 +0000 Subject: [PATCH 45/47] fix: remove outdated comment --- pkg/apis/v1alpha2/aksnodeclass.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index 4c06bb6a2..0846ea52a 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -144,7 +144,6 @@ type KubeletConfiguration struct { PodPidsLimit *int64 `json:"podPidsLimit,omitempty"` } -// TODO: add hashing support // AKSNodeClass is the Schema for the AKSNodeClass API // +kubebuilder:object:root=true // +kubebuilder:resource:path=aksnodeclasses,scope=Cluster,categories=karpenter,shortName={aksnc,aksncs} From 3569700055ca4789c4d0bb626fbce457f748bb07 Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:45:22 +0000 Subject: [PATCH 46/47] fix: typo --- pkg/apis/v1alpha2/aksnodeclass.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/apis/v1alpha2/aksnodeclass.go b/pkg/apis/v1alpha2/aksnodeclass.go index 0846ea52a..49f3764df 100644 --- a/pkg/apis/v1alpha2/aksnodeclass.go +++ b/pkg/apis/v1alpha2/aksnodeclass.go @@ -88,7 +88,7 @@ type KubeletConfiguration struct { // garbage collection is always run. The percent is calculated by dividing this // field value by 100, so this field must be between 0 and 100, inclusive. // When specified, the value must be greater than ImageGCLowThresholdPercent. - // Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name + // Note: AKS CustomKubeletConfig does not have "Percent" in the field name // +kubebuilder:validation:Minimum:=0 // +kubebuilder:validation:Maximum:=100 // +optional From c91ea43bf16597b25e6133fffeef1d2dfbb9ec7a Mon Sep 17 00:00:00 2001 From: tallaxes <18728999+tallaxes@users.noreply.github.com> Date: Thu, 24 Oct 2024 21:56:29 +0000 Subject: [PATCH 47/47] chore: update CRDs --- .../templates/karpenter.azure.com_aksnodeclasses.yaml | 2 +- pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml index e3c7878d9..fcf37a76d 100644 --- a/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.azure.com_aksnodeclasses.yaml @@ -111,7 +111,7 @@ spec: garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. - Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name + Note: AKS CustomKubeletConfig does not have "Percent" in the field name format: int32 maximum: 100 minimum: 0 diff --git a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml index e3c7878d9..fcf37a76d 100644 --- a/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml +++ b/pkg/apis/crds/karpenter.azure.com_aksnodeclasses.yaml @@ -111,7 +111,7 @@ spec: garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. - Note: AKS AKS CustomKubeletConfig does not have "Percent" in the field name + Note: AKS CustomKubeletConfig does not have "Percent" in the field name format: int32 maximum: 100 minimum: 0