diff --git a/.github/workflows/docs-vale.yml b/.github/workflows/docs-vale.yml index 505b291784..a1cfc4cbd9 100644 --- a/.github/workflows/docs-vale.yml +++ b/.github/workflows/docs-vale.yml @@ -19,7 +19,11 @@ jobs: uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }} - + # Work around https://github.com/errata-ai/vale-action/issues/128. + - run: | + venv="$HOME/.local/share/venv" + python3 -m venv "$venv" + echo "$venv/bin" >> "$GITHUB_PATH" - name: Vale uses: errata-ai/vale-action@91ac403e8d26f5aa1b3feaa86ca63065936a85b6 # tag=reviewdog with: diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index a6ac4fa3f7..5cd837a050 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.28"] # should be default + kubernetesVersion: ["1.30"] # This should correspond to the current default k8s minor. attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] test: ["sonobuoy quick"] diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index b9a27949c3..90940f4e00 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index 3338c13847..81f8b0a69d 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index 598c4091ae..251fac11c6 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -73,53 +73,53 @@ jobs: - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-22.04" clusterCreation: "cli" diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 585ffe6b90..1ea5de7728 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 29f0f32be7..f548bcccb0 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -89,53 +89,53 @@ jobs: - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" @@ -290,27 +290,27 @@ jobs: - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" runs-on: ubuntu-24.04 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c06c8eefff..1e413f48d6 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -43,7 +43,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" + default: "1.30" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel index ef87085bd8..968a2b0114 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "@io_k8s_kubelet//config/v1beta1", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", + "@org_golang_x_mod//semver", ], ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 430839a469..1ce387aeb4 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" + "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl cloudProvider = "external" } - return KubeadmInitYAML{ + initConfig := KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), @@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl TLSPrivateKeyFile: certificate.KeyFilename, }, } + + if semver.Compare(clusterVersion, "v1.31.0") >= 0 { + initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true} + } + return initConfig } // JoinConfiguration returns a new kubeadm join configuration. diff --git a/docs/docs/architecture/versions.md b/docs/docs/architecture/versions.md index 30d9d28e24..59f9785fff 100644 --- a/docs/docs/architecture/versions.md +++ b/docs/docs/architecture/versions.md @@ -16,6 +16,6 @@ Subsequent Constellation releases drop support for the oldest (and deprecated) K The following Kubernetes versions are currently supported: -* v1.28.13 * v1.29.8 * v1.30.4 +* v1.31.1 diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index 99acef5203..ebddf84c9a 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -80,7 +80,7 @@ constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] ``` -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used -h, --help help for generate - -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") -t, --tags strings additional tags for created resources given a list of key=value ``` diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml index 18dce5e376..9156e3e714 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/autoscalingstrategy-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: autoscalingstrategies.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: autoscalingstrategies.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -20,14 +21,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -48,8 +54,8 @@ spec: deployment. type: string enabled: - description: Enabled defines whether cluster autoscaling should be enabled - or not. + description: Enabled defines whether cluster autoscaling should be + enabled or not. type: boolean required: - deploymentName @@ -64,7 +70,8 @@ spec: enabled or not. type: boolean replicas: - description: Replicas is the number of replicas for the autoscaler deployment. + description: Replicas is the number of replicas for the autoscaler + deployment. format: int32 type: integer type: object @@ -73,9 +80,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml index 88fb65ae81..1beca72211 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/joiningnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: joiningnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: joiningnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: JoiningNode is the Schema for the joiningnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -59,9 +65,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] \ No newline at end of file diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml index 9c46b695cd..04693de2e8 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/nodeversion-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: nodeversions.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: nodeversions.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: NodeVersion is the Schema for the nodeversions API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -48,6 +54,11 @@ spec: description: KubernetesComponentsReference is a reference to the ConfigMap containing the Kubernetes components to use for all nodes. type: string + maxNodeBudget: + description: MaxNodeBudget is the maximum number of nodes that can + be created simultaneously. + format: int32 + type: integer type: object status: description: NodeVersionStatus defines the observed state of NodeVersion. @@ -60,65 +71,49 @@ spec: description: AwaitingAnnotation is a list of nodes that are waiting for the operator to annotate them. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array budget: description: Budget is the amount of extra nodes that can be created @@ -129,43 +124,35 @@ spec: description: Conditions represent the latest available observations of an object's state items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -180,10 +167,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -199,516 +182,389 @@ spec: description: Donors is a list of outdated nodes that donate labels to heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array heirs: description: Heirs is a list of nodes using the latest image that still need to inherit labels from donors. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array invalid: description: Invalid is a list of invalid nodes (nodes that cannot be processed by the operator due to missing information or transient faults). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array mints: description: Mints is a list of up to date nodes that will become heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array obsolete: description: Obsolete is a list of obsolete nodes (nodes that have been created by the operator but are no longer needed). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array outdated: description: Outdated is a list of nodes that are using an outdated image. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array pending: description: Pending is a list of pending nodes (joining or leaving the cluster). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array upToDate: description: UpToDate is a list of nodes that are using the latest image and labels. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array required: + - activeclusterversionupgrade - budget - conditions type: object diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml index 41b5a4cd72..c6cd2db6ac 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/pendingnode-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: pendingnodes.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: pendingnodes.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: PendingNode is the Schema for the pendingnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -34,10 +40,11 @@ spec: description: PendingNodeSpec defines the desired state of PendingNode. properties: deadline: - description: Deadline is the deadline for reaching the goal state. Joining - nodes will be terminated if the deadline is exceeded. Leaving nodes - will remain as unschedulable to prevent data loss. If not specified, - the node may remain in the pending state indefinitely. + description: |- + Deadline is the deadline for reaching the goal state. + Joining nodes will be terminated if the deadline is exceeded. + Leaving nodes will remain as unschedulable to prevent data loss. + If not specified, the node may remain in the pending state indefinitely. format: date-time type: string goal: @@ -47,8 +54,8 @@ spec: - Leave type: string groupID: - description: ScalingGroupID is the ID of the group that this node shall - be part of. + description: ScalingGroupID is the ID of the group that this node + shall be part of. type: string nodeName: description: NodeName is the kubernetes internal name of the node. @@ -72,7 +79,8 @@ spec: - Failed type: string reachedGoal: - description: ReachedGoal is true if the node has reached the goal state. + description: ReachedGoal is true if the node has reached the goal + state. type: boolean type: object type: object @@ -80,9 +88,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml index 0e334ae297..5eed4ebc88 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/crds/scalinggroup-crd.yaml @@ -1,9 +1,10 @@ +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: - name: scalinggroups.update.edgeless.systems annotations: - controller-gen.kubebuilder.io/version: v0.9.0 + controller-gen.kubebuilder.io/version: v0.16.4 + name: scalinggroups.update.edgeless.systems spec: group: update.edgeless.systems names: @@ -19,14 +20,19 @@ spec: description: ScalingGroup is the Schema for the scalinggroups API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -37,16 +43,16 @@ spec: description: AutoscalerGroupName is name that is expected by the autoscaler. type: string autoscaling: - description: Autoscaling specifies wether the scaling group should automatically - scale using the cluster-autoscaler. + description: Autoscaling specifies wether the scaling group should + automatically scale using the cluster-autoscaler. type: boolean groupId: - description: GroupID is the CSP specific, canonical identifier of a - scaling group. + description: GroupID is the CSP specific, canonical identifier of + a scaling group. type: string max: - description: Max is the maximum number of autoscaled nodes in the scaling - group (used by cluster-autoscaler). + description: Max is the maximum number of autoscaled nodes in the + scaling group (used by cluster-autoscaler). format: int32 type: integer min: @@ -55,11 +61,11 @@ spec: format: int32 type: integer nodeGroupName: - description: NodeGroupName is the human friendly name of the node group - as defined in the Constellation configuration. + description: NodeGroupName is the human friendly name of the node + group as defined in the Constellation configuration. type: string nodeImage: - description: NodeImage is the name of the NodeImage resource. + description: NodeVersion is the name of the NodeVersion resource. type: string role: description: Role is the role of the nodes in the scaling group. @@ -75,44 +81,36 @@ spec: description: Conditions represent the latest available observations of an object's state. items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a foo's - current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating details - about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers of - specific condition types may define expected values and meanings - for this field, and whether the values are considered a guaranteed - API. The value should be a CamelCase string. This field may - not be empty. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. maxLength: 1024 minLength: 1 pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ @@ -126,10 +124,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -142,8 +136,8 @@ spec: type: object type: array imageReference: - description: ImageReference is the image currently used for newly created - nodes in this scaling group. + description: ImageReference is the image currently used for newly + created nodes in this scaling group. type: string required: - conditions @@ -153,9 +147,3 @@ spec: storage: true subresources: status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml index 45dddbdd92..b75dfed440 100644 --- a/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/charts/edgeless/operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -58,98 +58,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -161,13 +73,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -175,29 +95,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/AWS/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/Azure/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/GCP/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/OpenStack/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml index 4fa4863c80..03b13a202f 100644 --- a/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml +++ b/internal/constellation/helm/testdata/QEMU/constellation-operators/charts/constellation-operator/templates/manager-rbac.yaml @@ -61,98 +61,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -164,13 +76,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -178,29 +98,17 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/internal/constellation/kubecmd/BUILD.bazel b/internal/constellation/kubecmd/BUILD.bazel index 71bae3c32d..aca26d0bb2 100644 --- a/internal/constellation/kubecmd/BUILD.bazel +++ b/internal/constellation/kubecmd/BUILD.bazel @@ -30,8 +30,11 @@ go_library( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/runtime/serializer/json", "@io_k8s_client_go//util/retry", - "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/scheme", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta4", "@io_k8s_sigs_yaml//:yaml", ], ) diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index 1ebf992654..dd3c4f9da7 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -42,9 +42,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/util/retry" - kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - "sigs.k8s.io/yaml" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmv1beta4 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" ) // ErrInProgress signals that an upgrade is in progress inside the cluster. @@ -129,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio ) } + // TODO(burgerdev): remove after releasing v2.19 + // Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to + // connect to the local API server. + if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) { + if cc.FeatureGates == nil { + cc.FeatureGates = map[string]bool{} + } + cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true + }); err != nil { + return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err) + } + versionConfig, ok := versions.VersionConfigs[kubernetesVersion] if !ok { return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( @@ -234,48 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) - if err != nil { - return fmt.Errorf("getting ClusterConfig: %w", err) - } - - existingSANs := make(map[string]struct{}) - for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { - existingSANs[existingSAN] = struct{}{} - } - - var missingSANs []string - for _, san := range alternativeNames { - if san == "" { - continue // skip empty SANs + if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) { + existingSANs := make(map[string]struct{}) + for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { + existingSANs[existingSAN] = struct{}{} } - if _, ok := existingSANs[san]; !ok { - missingSANs = append(missingSANs, san) - existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice - } - } - - if len(missingSANs) == 0 { - k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") - return nil - } - k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) - sort.Strings(clusterConfiguration.APIServer.CertSANs) + var missingSANs []string + for _, san := range alternativeNames { + if san == "" { + continue // skip empty SANs + } + if _, ok := existingSANs[san]; !ok { + missingSANs = append(missingSANs, san) + existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + } + } - newConfigYAML, err := yaml.Marshal(clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } + if len(missingSANs) == 0 { + k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") + } + k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debug("Triggering kubeadm config update now") - if err = k.retryAction(ctx, func(ctx context.Context) error { - _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) - return err + clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) + sort.Strings(clusterConfiguration.APIServer.CertSANs) }); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) + return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err) } k.log.Debug("Successfully extended the cluster's apiserver SAN field") @@ -316,31 +314,6 @@ func (k *KubeCmd) getConstellationVersion(ctx context.Context) (updatev1alpha1.N return nodeVersion, nil } -// getClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config -// and returns both the full configmap and the ClusterConfiguration. -func (k *KubeCmd) getClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) { - var existingConf *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - existingConf, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey] - if !ok { - return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var existingClusterConfig kubeadmv1beta3.ClusterConfiguration - if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err) - } - - return existingClusterConfig, existingConf, nil -} - // applyComponentsCM applies the k8s components ConfigMap to the cluster. func (k *KubeCmd) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error { if err := k.retryAction(ctx, func(ctx context.Context) error { @@ -468,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte return retrier.Do(ctx) } +// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry, +// runs doPatch on it and uploads the result. +func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error { + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) + } + + doPatch(&clusterConfiguration) + + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debug("Triggering kubeadm config update now") + if err = k.retryAction(ctx, func(ctx context.Context) error { + _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) + return err + }); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + k.log.Debug("Successfully patched the cluster's kubeadm-config") + return nil +} + func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error { var err error switch { diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 74e9562c15..c88dab9bda 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) { } kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}}, + }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) @@ -676,6 +679,50 @@ func TestRetryAction(t *testing.T) { } } +func TestExtendClusterConfigCertSANs(t *testing.T) { + ctx := context.Background() + + testCases := map[string]struct { + clusterConfig string + }{ + "kubeadmv1beta3.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta3, + }, + "kubeadmv1beta4.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta4, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + kubectl := &fakeConfigMapClient{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": tc.clusterConfig}}, + }, + } + cmd := &KubeCmd{ + kubectl: kubectl, + log: logger.NewTest(t), + retryInterval: time.Millisecond, + } + + err := cmd.ExtendClusterConfigCertSANs(ctx, []string{"example.com"}) + require.NoError(err) + + cm := kubectl.configMaps["kubeadm-config"] + require.NotNil(cm) + cc := cm.Data["ClusterConfiguration"] + require.NotNil(cc) + // Verify that SAN was added. + assert.Contains(cc, "example.com") + // Verify that config was written in v1beta4, regardless of the version read. + assert.Contains(cc, "kubeadm.k8s.io/v1beta4") + }) + } +} + type fakeUnstructuredClient struct { mock.Mock } @@ -835,3 +882,83 @@ func supportedValidK8sVersions() (res []versions.ValidK8sVersion) { } return } + +var kubeadmClusterConfigurationV1Beta3 = ` +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt + profiling: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + cloud-provider: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + profiling: "false" +` + +var kubeadmClusterConfigurationV1Beta4 = ` +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + - name: kubelet-certificate-authority + value: /etc/kubernetes/pki/ca.crt + - name: profiling + value: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + - name: cloud-provider + value: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + - name: profiling + value: "false" +` diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 0621b5d744..fe600bca23 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -100,12 +100,12 @@ func ResolveK8sPatchVersion(k8sVersion string) (string, error) { // supported patch version as PATCH. func k8sVersionFromMajorMinor(version string) string { switch version { - case semver.MajorMinor(string(V1_28)): - return string(V1_28) case semver.MajorMinor(string(V1_29)): return string(V1_29) case semver.MajorMinor(string(V1_30)): return string(V1_30) + case semver.MajorMinor(string(V1_31)): + return string(V1_31) default: return "" } @@ -181,14 +181,14 @@ const ( // currently supported versions. //nolint:revive - V1_28 ValidK8sVersion = "v1.28.13" // renovate:kubernetes-release - //nolint:revive V1_29 ValidK8sVersion = "v1.29.8" // renovate:kubernetes-release //nolint:revive V1_30 ValidK8sVersion = "v1.30.4" // renovate:kubernetes-release + //nolint:revive + V1_31 ValidK8sVersion = "v1.31.1" // renovate:kubernetes-release // Default k8s version deployed by Constellation. - Default ValidK8sVersion = V1_29 + Default ValidK8sVersion = V1_30 ) // Regenerate the hashes by running go generate. @@ -197,8 +197,8 @@ const ( // VersionConfigs holds download URLs for all required kubernetes components for every supported version. var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ - V1_28: { - ClusterVersion: "v1.28.13", // renovate:kubernetes-release + V1_29: { + ClusterVersion: "v1.29.8", // renovate:kubernetes-release KubernetesComponents: components.Components{ { Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release @@ -207,39 +207,39 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.28.0/crictl-v1.28.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:8dc78774f7cbeaf787994d386eec663f0a3cf24de1ea4893598096cb39ef2508", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:9b9cc3a19551ade6f3d98ad3acf0a2b65a27ef575bd089f115f8bb80791f3900", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:df6e130928403af8b4f49f1197e26f2873a147cd0e23aa6597a26c982c652ae0", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:f23e9586811312998bc5e8847f6df52fc04809aed8c2c2fd750f2c42b3f87192", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:fe054355e0ae8dc35d868a3d3bc408ccdff0969c20bf7a231ae9b71484e41be3", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.28.13/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:d7d363dd5a4c95444329bc5239b8718ebe84a043052958b2f15ee2feef9a28c6", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:038454e0d79748aab41668f44ca6e4ac8affd1895a94f592b9739a0ae2a5f06a", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI4LjEzQHNoYTI1Njo3ZDJjOTI1NmFkNTc2YTBiMzc0NWI3NDllZmU3ZjRmYThiMjc2ZWM3ZWY0NDhmYzBmNDU3OTRjYTc4ZWI4NjI1In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjhAc2hhMjU2OjZmNzJmYTkyNmM5YjA1ZTEwNjI5ZmUxYTA5MmZkMjhkY2Q2NWI0ZmRmZDBjYzdiZDU1Zjg1YTU3YTZiYTFmYTUifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI4LjEzQHNoYTI1NjplN2I0NGMxNzQxZmUxODAyZDE1OWZmZGJkMGQxZjc4ZDQ4YTQxODVkN2ZiMWNkZjhhMTEyZmJiNTA2OTZmN2UxIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjhAc2hhMjU2OjZmMjdkNjNkZWQyMDYxNGM2ODU1NGI0NzdjZDdhNzhlZGE3OGE0OThhOTJiZmU4OTM1Y2Y5NjRjYTViNzRkMGIifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI4LjEzQHNoYTI1NjplZmViNzkxNzE4ZjRiOWM2MmJkNjgzZjViNDAzZGE1MjBmMzY1MWNiMzZhZDlmODAwZTBmOThiNTk1YmVhZmE0In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjhAc2hhMjU2OmRhNzRhNjY2NzVkOTVlMzllYzI1ZGE1ZTcwNzI5ZGE3NDZkMGZhMGIxNWVlMGRhODcyYWM5ODA1MTliYzI4YmQifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -249,23 +249,23 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.9@sha256:168905b591796fbd07cb35cd0e3f206fdb7efb30e325c9bf7fa70d1b48989f73", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.6@sha256:8074b8828a33fb273833e8fd374dda6a0ab10335ae8e19684fbd61eeff7d3594", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.28.12@sha256:5ba85a312fdb6b65d4267a1e42090aaa9ebb69f44858d9f9f806a317a9260530", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.10@sha256:2e4d2c0bb15f2987cb7abb52e86c9b62504226fdbf19d5b597ac4707adb52a56", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.28.12@sha256:e4a4ddb678f625399b85f4becff16ef07a06da7f5f4673ca21cfb356c370930d", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.10@sha256:ef27f4e2c104614fa44265d6e0a193bf1f9b5abda542f5b64efb868d3796ffc8", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v28.10.0@sha256:f3b6fa7faea27b4a303c91b3bc7ee192b050e21e27579e9f3da90ae4ba38e626", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.6@sha256:acfc7fe7543f4cf2fcf8156145925bee76eb6c602bb0b8e155456c6818fe8335", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.4@sha256:786728c85787a58c6376b47d2e22cc04db3ecfdd73a52b5b9be20fd869abce2f", // renovate:container }, - V1_29: { - ClusterVersion: "v1.29.8", // renovate:kubernetes-release + V1_30: { + ClusterVersion: "v1.30.4", // renovate:kubernetes-release KubernetesComponents: components.Components{ { Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release @@ -274,39 +274,39 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.29.0/crictl-v1.29.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:d16a1ffb3938f5a19d5c8f45d363bd091ef89c0bc4d44ad16b933eede32fdcbb", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.30.0/crictl-v1.30.0-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:3dd03954565808eaeb3a7ffc0e8cb7886a64a9aa94b2bfdfbdc6e2ed94842e49", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:df6e130928403af8b4f49f1197e26f2873a147cd0e23aa6597a26c982c652ae0", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:0c02c0f997b3e9769eae7ca051856054411fca947b3d5409d991ce1964dd0e69", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:fe054355e0ae8dc35d868a3d3bc408ccdff0969c20bf7a231ae9b71484e41be3", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:6c6053fb8b31030ef7fffe146eb29489f7bf53d7a5ca10e0b10c907bf4b7e281", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.29.8/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:038454e0d79748aab41668f44ca6e4ac8affd1895a94f592b9739a0ae2a5f06a", + Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:2ffd023712bbc1a9390dbd8c0c15201c165a69d394787ef03eda3eccb4b9ac06", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjhAc2hhMjU2OjZmNzJmYTkyNmM5YjA1ZTEwNjI5ZmUxYTA5MmZkMjhkY2Q2NWI0ZmRmZDBjYzdiZDU1Zjg1YTU3YTZiYTFmYTUifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjRAc2hhMjU2OjdiMGM0YTk1OWFhZWU1NjYwZTEyMzQ0NTJkYzMxMjMzMTAyMzFiOWY5MmQyOWViZDE3NWM4NmRjOWY3OTdlZTcifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjhAc2hhMjU2OjZmMjdkNjNkZWQyMDYxNGM2ODU1NGI0NzdjZDdhNzhlZGE3OGE0OThhOTJiZmU4OTM1Y2Y5NjRjYTViNzRkMGIifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjRAc2hhMjU2Ojk5MmNjY2JmNjUyZmE5NTFjMWEzZDQxYjBjMTAzM2FlMGJmNjRmMzNkYTAzZDUwMzk1MjgyYzU1MTkwMGFmOWUifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjhAc2hhMjU2OmRhNzRhNjY2NzVkOTVlMzllYzI1ZGE1ZTcwNzI5ZGE3NDZkMGZhMGIxNWVlMGRhODcyYWM5ODA1MTliYzI4YmQifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjRAc2hhMjU2OjM3ZWFlZWU1YmNhOGRhMzRhZDNkMzZlMzc1ODZkZDI5ZjVlZGIxZTI5MjdlNzY0NGRmYjExM2U3MDA2MmJkYTgifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -316,64 +316,64 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.6@sha256:8074b8828a33fb273833e8fd374dda6a0ab10335ae8e19684fbd61eeff7d3594", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3@sha256:30a1758dec30814178c787e2d50f46bb141e9f0bb2e16190ddd19df15f957874", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.10@sha256:2e4d2c0bb15f2987cb7abb52e86c9b62504226fdbf19d5b597ac4707adb52a56", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.6@sha256:4cbf44b038ff426dccb457b160f7c675d736a062ec9e729996c568bf3d252c67", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.10@sha256:ef27f4e2c104614fa44265d6e0a193bf1f9b5abda542f5b64efb868d3796ffc8", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.6@sha256:89ea8b66d026fe313f7f5781d1be6c44bf2c345ead89b8fe256c2bfab47304d1", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.4@sha256:786728c85787a58c6376b47d2e22cc04db3ecfdd73a52b5b9be20fd869abce2f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.2@sha256:ef370d1d06a1603c4b1e47e64bb27c3ff51cb20680712dc3d41c34f3fbf7be9f", // renovate:container }, - V1_30: { - ClusterVersion: "v1.30.4", // renovate:kubernetes-release + V1_31: { + ClusterVersion: "v1.31.1", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.4.0/cni-plugins-linux-amd64-v1.4.0.tgz", // renovate:cni-plugins-release - Hash: "sha256:c2485ddb3ffc176578ae30ae58137f0b88e50f7c7f2af7d53a569276b2949a33", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release + Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.30.0/crictl-v1.30.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:3dd03954565808eaeb3a7ffc0e8cb7886a64a9aa94b2bfdfbdc6e2ed94842e49", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:0c02c0f997b3e9769eae7ca051856054411fca947b3d5409d991ce1964dd0e69", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:6c6053fb8b31030ef7fffe146eb29489f7bf53d7a5ca10e0b10c907bf4b7e281", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://storage.googleapis.com/kubernetes-release/release/v1.30.4/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:2ffd023712bbc1a9390dbd8c0c15201c165a69d394787ef03eda3eccb4b9ac06", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjRAc2hhMjU2OjdiMGM0YTk1OWFhZWU1NjYwZTEyMzQ0NTJkYzMxMjMzMTAyMzFiOWY5MmQyOWViZDE3NWM4NmRjOWY3OTdlZTcifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjRAc2hhMjU2Ojk5MmNjY2JmNjUyZmE5NTFjMWEzZDQxYjBjMTAzM2FlMGJmNjRmMzNkYTAzZDUwMzk1MjgyYzU1MTkwMGFmOWUifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjRAc2hhMjU2OjM3ZWFlZWU1YmNhOGRhMzRhZDNkMzZlMzc1ODZkZDI5ZjVlZGIxZTI5MjdlNzY0NGRmYjExM2U3MDA2MmJkYTgifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -383,20 +383,20 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.3@sha256:30a1758dec30814178c787e2d50f46bb141e9f0bb2e16190ddd19df15f957874", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.0@sha256:c4658f5c583e85ef58a096ac9901a8ced3cb9699d8eeceb59d3bb54c5d0db72f", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.6@sha256:4cbf44b038ff426dccb457b160f7c675d736a062ec9e729996c568bf3d252c67", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.6@sha256:89ea8b66d026fe313f7f5781d1be6c44bf2c345ead89b8fe256c2bfab47304d1", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container + CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.2@sha256:ef370d1d06a1603c4b1e47e64bb27c3ff51cb20680712dc3d41c34f3fbf7be9f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container }, } diff --git a/operators/constellation-node-operator/Makefile b/operators/constellation-node-operator/Makefile index ca2bd09068..777b1ecb1a 100644 --- a/operators/constellation-node-operator/Makefile +++ b/operators/constellation-node-operator/Makefile @@ -86,11 +86,11 @@ help: ## Display this help. .PHONY: manifests manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./" output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: generate generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./" + $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." .PHONY: fmt fmt: ## Run go fmt against code. @@ -162,7 +162,7 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ## Tool Versions KUSTOMIZE_VERSION ?= v3.8.7 -CONTROLLER_TOOLS_VERSION ?= v0.9.0 +CONTROLLER_TOOLS_VERSION ?= v0.16.4 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" .PHONY: kustomize diff --git a/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go b/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go index f5d81dfede..6e741c7d83 100644 --- a/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go +++ b/operators/constellation-node-operator/api/v1alpha1/nodeversion_types.go @@ -21,6 +21,8 @@ type NodeVersionSpec struct { KubernetesComponentsReference string `json:"kubernetesComponentsReference,omitempty"` // KubernetesClusterVersion is the advertised Kubernetes version of the cluster. KubernetesClusterVersion string `json:"kubernetesClusterVersion,omitempty"` + // MaxNodeBudget is the maximum number of nodes that can be created simultaneously. + MaxNodeBudget uint32 `json:"maxNodeBudget,omitempty"` } // NodeVersionStatus defines the observed state of NodeVersion. diff --git a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go index 198fc055ce..b25cfc280f 100644 --- a/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operators/constellation-node-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated // Code generated by controller-gen. DO NOT EDIT. diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml index 6a014a3947..9156e3e714 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_autoscalingstrategies.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: autoscalingstrategies.update.edgeless.systems spec: group: update.edgeless.systems @@ -22,14 +21,19 @@ spec: API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml index 9c8e9e5a29..1beca72211 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_joiningnodes.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: joiningnodes.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: JoiningNode is the Schema for the joiningnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml index e4c435ec1c..04693de2e8 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_nodeversions.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: nodeversions.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: NodeVersion is the Schema for the nodeversions API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -50,6 +54,11 @@ spec: description: KubernetesComponentsReference is a reference to the ConfigMap containing the Kubernetes components to use for all nodes. type: string + maxNodeBudget: + description: MaxNodeBudget is the maximum number of nodes that can + be created simultaneously. + format: int32 + type: integer type: object status: description: NodeVersionStatus defines the observed state of NodeVersion. @@ -62,65 +71,49 @@ spec: description: AwaitingAnnotation is a list of nodes that are waiting for the operator to annotate them. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array budget: description: Budget is the amount of extra nodes that can be created @@ -131,43 +124,35 @@ spec: description: Conditions represent the latest available observations of an object's state items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -182,10 +167,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string @@ -201,514 +182,386 @@ spec: description: Donors is a list of outdated nodes that donate labels to heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array heirs: description: Heirs is a list of nodes using the latest image that still need to inherit labels from donors. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array invalid: description: Invalid is a list of invalid nodes (nodes that cannot be processed by the operator due to missing information or transient faults). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array mints: description: Mints is a list of up to date nodes that will become heirs. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array obsolete: description: Obsolete is a list of obsolete nodes (nodes that have been created by the operator but are no longer needed). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array outdated: description: Outdated is a list of nodes that are using an outdated image. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array pending: description: Pending is a list of pending nodes (joining or leaving the cluster). items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array upToDate: description: UpToDate is a list of nodes that are using the latest image and labels. items: - description: "ObjectReference contains enough information to let - you inspect or modify the referred object. --- New uses of this - type are discouraged because of difficulty describing its usage - when embedded in APIs. 1. Ignored fields. It includes many fields - which are not generally honored. For instance, ResourceVersion - and FieldPath are both very rarely valid in actual usage. 2. Invalid - usage help. It is impossible to add specific help for individual - usage. In most embedded usages, there are particular restrictions - like, \"must refer only to types A and B\" or \"UID not honored\" - or \"name must be restricted\". Those cannot be well described - when embedded. 3. Inconsistent validation. Because the usages - are different, the validation rules are different by usage, which - makes it hard for users to predict what will happen. 4. The fields - are both imprecise and overly precise. Kind is not a precise - mapping to a URL. This can produce ambiguity during interpretation - and require a REST mapping. In most cases, the dependency is - on the group,resource tuple and the version of the actual struct - is irrelevant. 5. We cannot easily change it. Because this type - is embedded in many locations, updates to this type will affect - numerous schemas. Don't make new APIs embed an underspecified - API type they do not control. \n Instead of using this type, create - a locally provided and used type that is well-focused on your - reference. For example, ServiceReferences for admission registration: - https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 - ." + description: ObjectReference contains enough information to let + you inspect or modify the referred object. properties: apiVersion: description: API version of the referent. type: string fieldPath: - description: 'If referring to a piece of an object instead of - an entire object, this string should contain a valid JSON/Go - field access statement, such as desiredState.manifest.containers[2]. - For example, if the object reference is to a container within - a pod, this would take on a value like: "spec.containers{name}" - (where "name" refers to the name of the container that triggered - the event) or if no container name is specified "spec.containers[2]" - (container with index 2 in this pod). This syntax is chosen - only to have some well-defined way of referencing a part of - an object. TODO: this design is not final and this field is - subject to change in the future.' + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. type: string kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names type: string namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ type: string resourceVersion: - description: 'Specific resourceVersion to which this reference - is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency type: string uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids type: string type: object + x-kubernetes-map-type: atomic type: array required: - activeclusterversionupgrade diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml index 7c5b5618b5..c6cd2db6ac 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_pendingnodes.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: pendingnodes.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: PendingNode is the Schema for the pendingnodes API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -36,10 +40,11 @@ spec: description: PendingNodeSpec defines the desired state of PendingNode. properties: deadline: - description: Deadline is the deadline for reaching the goal state. - Joining nodes will be terminated if the deadline is exceeded. Leaving - nodes will remain as unschedulable to prevent data loss. If not - specified, the node may remain in the pending state indefinitely. + description: |- + Deadline is the deadline for reaching the goal state. + Joining nodes will be terminated if the deadline is exceeded. + Leaving nodes will remain as unschedulable to prevent data loss. + If not specified, the node may remain in the pending state indefinitely. format: date-time type: string goal: diff --git a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml index 0f87fbab1c..5eed4ebc88 100644 --- a/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml +++ b/operators/constellation-node-operator/config/crd/bases/update.edgeless.systems_scalinggroups.yaml @@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.9.0 - creationTimestamp: null + controller-gen.kubebuilder.io/version: v0.16.4 name: scalinggroups.update.edgeless.systems spec: group: update.edgeless.systems @@ -21,14 +20,19 @@ spec: description: ScalingGroup is the Schema for the scalinggroups API. properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -57,8 +61,8 @@ spec: format: int32 type: integer nodeGroupName: - description: NodeGroupName is the human friendly name of the node group - as defined in the Constellation configuration. + description: NodeGroupName is the human friendly name of the node + group as defined in the Constellation configuration. type: string nodeImage: description: NodeVersion is the name of the NodeVersion resource. @@ -77,43 +81,35 @@ spec: description: Conditions represent the latest available observations of an object's state. items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - \n type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: \"Available\", - \"Progressing\", and \"Degraded\" // +patchMergeKey=type // +patchStrategy=merge - // +listType=map // +listMapKey=type Conditions []metav1.Condition - `json:\"conditions,omitempty\" patchStrategy:\"merge\" patchMergeKey:\"type\" - protobuf:\"bytes,1,rep,name=conditions\"` \n // other fields }" + description: Condition contains details for one aspect of the current + state of this API Resource. properties: lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. format: date-time type: string message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. maxLength: 32768 type: string observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. format: int64 minimum: 0 type: integer reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. This field may not be empty. maxLength: 1024 minLength: 1 @@ -128,10 +124,6 @@ spec: type: string type: description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) maxLength: 316 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ type: string diff --git a/operators/constellation-node-operator/config/manager/kustomization.yaml b/operators/constellation-node-operator/config/manager/kustomization.yaml index 23ff3a7e4c..6919d782b9 100644 --- a/operators/constellation-node-operator/config/manager/kustomization.yaml +++ b/operators/constellation-node-operator/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: ghcr.io/edgelesssys/constellation/node-operator - newTag: v0.0.0 + newTag: v0.0.1 diff --git a/operators/constellation-node-operator/config/rbac/role.yaml b/operators/constellation-node-operator/config/rbac/role.yaml index 8700132c49..169011983a 100644 --- a/operators/constellation-node-operator/config/rbac/role.yaml +++ b/operators/constellation-node-operator/config/rbac/role.yaml @@ -2,7 +2,6 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: - creationTimestamp: null name: manager-role rules: - apiGroups: @@ -57,98 +56,10 @@ rules: - update.edgeless.systems resources: - autoscalingstrategies - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - autoscalingstrategies/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - joiningnodes - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - joiningnodes/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion - verbs: - - get - - list - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversion/status - verbs: - - get -- apiGroups: - - update.edgeless.systems - resources: - nodeversions - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - nodeversions/status - verbs: - - get - - patch - - update -- apiGroups: - - update.edgeless.systems - resources: - pendingnodes + - scalinggroups verbs: - create - delete @@ -160,13 +71,21 @@ rules: - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/finalizers + - joiningnodes/finalizers + - nodeversions/finalizers - pendingnodes/finalizers + - scalinggroups/finalizers verbs: - update - apiGroups: - update.edgeless.systems resources: + - autoscalingstrategies/status + - joiningnodes/status + - nodeversions/status - pendingnodes/status + - scalinggroups/status verbs: - get - patch @@ -174,26 +93,14 @@ rules: - apiGroups: - update.edgeless.systems resources: - - scalinggroups + - nodeversion verbs: - - create - - delete - get - list - - patch - - update - watch - apiGroups: - update.edgeless.systems resources: - - scalinggroups/finalizers - verbs: - - update -- apiGroups: - - update.edgeless.systems - resources: - - scalinggroups/status + - nodeversion/status verbs: - get - - patch - - update diff --git a/operators/constellation-node-operator/controllers/BUILD.bazel b/operators/constellation-node-operator/controllers/BUILD.bazel index f665358091..2a24c2584b 100644 --- a/operators/constellation-node-operator/controllers/BUILD.bazel +++ b/operators/constellation-node-operator/controllers/BUILD.bazel @@ -81,6 +81,7 @@ go_test( "//3rdparty/node-maintenance-operator/api/v1beta1", "//internal/constants", "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@com_github_onsi_ginkgo_v2//:ginkgo", "@com_github_onsi_gomega//:gomega", "@com_github_stretchr_testify//assert", diff --git a/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go index 2662dff284..4dfa2907b4 100644 --- a/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/joiningnode_controller_env_test.go @@ -39,6 +39,13 @@ var _ = Describe("JoiningNode controller", func() { duration = time.Second * 2 interval = time.Millisecond * 250 ) + + AfterEach(func() { + Eventually(func() error { + return resetEnv() + }, 30*time.Second, 1*time.Second).Should(Succeed()) + }) + Context("When changing a joining node resource spec", func() { It("Should annotate the corresponding node when creating the CRD first", func() { By("creating a joining node resource") diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller.go b/operators/constellation-node-operator/controllers/nodeversion_controller.go index ff706c702f..a9508e5711 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller.go @@ -38,8 +38,6 @@ import ( ) const ( - // nodeOverprovisionLimit is the maximum number of extra nodes created during the update procedure at any point in time. - nodeOverprovisionLimit = 1 // nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated. nodeJoinTimeout = time.Minute * 30 // nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated. @@ -138,9 +136,9 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) logr.Error(err, "Unable to list scaling groups") return ctrl.Result{}, err } - scalingGroupByID := make(map[string]updatev1alpha1.ScalingGroup, len(scalingGroupList.Items)) + scalingGroupByID := make(scalingGroupByID, len(scalingGroupList.Items)) for _, scalingGroup := range scalingGroupList.Items { - scalingGroupByID[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup + scalingGroupByID.put(scalingGroup) } annotatedNodes, invalidNodes := r.annotateNodes(ctx, nodeList.Items) groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeVersion.Spec.ImageReference, desiredNodeVersion.Spec.KubernetesComponentsReference) @@ -161,9 +159,14 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) // - being created (joining) // - being destroyed (leaving) // - heirs to outdated nodes - extraNodes := len(groups.Heirs) + len(groups.AwaitingAnnotation) + len(pendingNodeList.Items) + extraNodes := uint32(len(groups.Heirs) + len(groups.AwaitingAnnotation) + len(pendingNodeList.Items)) // newNodesBudget is the maximum number of new nodes that can be created in this Reconcile call. - var newNodesBudget int + var newNodesBudget uint32 + nodeOverprovisionLimit := uint32(1) + // Use user definable value if it makes sense + if desiredNodeVersion.Spec.MaxNodeBudget > 0 { + nodeOverprovisionLimit = desiredNodeVersion.Spec.MaxNodeBudget + } if extraNodes < nodeOverprovisionLimit { newNodesBudget = nodeOverprovisionLimit - extraNodes } @@ -214,7 +217,7 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{Requeue: shouldRequeue}, nil } - newNodeConfig := newNodeConfig{desiredNodeVersion, groups.Outdated, pendingNodeList.Items, scalingGroupByID, newNodesBudget} + newNodeConfig := newNodeConfig{desiredNodeVersion, groups, pendingNodeList.Items, scalingGroupByID, newNodesBudget} if err := r.createNewNodes(ctx, newNodeConfig); err != nil { logr.Error(err, "Creating new nodes") return ctrl.Result{Requeue: shouldRequeue}, nil @@ -610,12 +613,21 @@ func (r *NodeVersionReconciler) deleteNode(ctx context.Context, controller metav // createNewNodes creates new nodes using up to date images as replacement for outdated nodes. func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNodeConfig) error { + hasOutdatedControlPlanes := func() bool { + for _, entry := range append(config.groups.Outdated, config.groups.Donors...) { + if nodeutil.IsControlPlaneNode(&entry) { + return true + } + } + return false + }() + logr := log.FromContext(ctx) - if config.newNodesBudget < 1 || len(config.outdatedNodes) == 0 { + if config.newNodesBudget < 1 || len(config.groups.Outdated) == 0 { return nil } outdatedNodesPerScalingGroup := make(map[string]int) - for _, node := range config.outdatedNodes { + for _, node := range config.groups.Outdated { // skip outdated nodes that got assigned an heir in this Reconcile call if len(node.Annotations[heirAnnotation]) != 0 { continue @@ -637,17 +649,22 @@ func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNo requiredNodesPerScalingGroup[scalingGroupID] = outdatedNodesPerScalingGroup[scalingGroupID] - pendingJoiningNodesPerScalingGroup[scalingGroupID] } } - for scalingGroupID := range requiredNodesPerScalingGroup { - scalingGroup, ok := config.scalingGroupByID[scalingGroupID] + for _, scalingGroupID := range config.scalingGroupByID.getScalingGroupIDsSorted() { + scalingGroup, ok := config.scalingGroupByID.get(scalingGroupID) if !ok { logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID, "scalingGroups", config.scalingGroupByID) continue } + if requiredNodesPerScalingGroup[scalingGroupID] == 0 { + logr.Info("No new nodes needed for scaling group", "scalingGroup", scalingGroupID) + continue + } if !strings.EqualFold(scalingGroup.Status.ImageReference, config.desiredNodeVersion.Spec.ImageReference) { logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", config.desiredNodeVersion.Spec.ImageReference) continue } - if requiredNodesPerScalingGroup[scalingGroupID] == 0 { + if hasOutdatedControlPlanes && scalingGroup.Spec.Role != updatev1alpha1.ControlPlaneRole { + logr.Info("There are still outdated control plane nodes which must be replaced first before this worker scaling group is upgraded", "scalingGroup", scalingGroupID) continue } for { @@ -676,10 +693,16 @@ func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNo if err := ctrl.SetControllerReference(&config.desiredNodeVersion, pendingNode, r.Scheme); err != nil { return err } + // Note that while we call Create here, it is not certain that the next reconciler iteration + // will see the pending node. This is because of delays in the kubeAPI. + // Currently, we don't explicitly wait until we can Get the resource. + // We can never be certain that other calls to Get will also see the resource, + // therefore it's just a tradeoff of the probability that we create more nodes than + // the specified budget. if err := r.Create(ctx, pendingNode); err != nil { return err } - logr.Info("Created new node", "createdNode", nodeName, "scalingGroup", scalingGroupID) + logr.Info("Created new node", "createdNode", nodeName, "scalingGroup", scalingGroupID, "requiredNodes", requiredNodesPerScalingGroup[scalingGroupID]) requiredNodesPerScalingGroup[scalingGroupID]-- config.newNodesBudget-- } @@ -743,7 +766,7 @@ func (r *NodeVersionReconciler) tryUpdateStatus(ctx context.Context, name types. } // nodeVersionStatus generates the NodeVersion.Status field given node groups and the budget for new nodes. -func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget int) updatev1alpha1.NodeVersionStatus { +func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget uint32) updatev1alpha1.NodeVersionStatus { var status updatev1alpha1.NodeVersionStatus outdatedCondition := metav1.Condition{ Type: updatev1alpha1.ConditionOutdated, @@ -821,7 +844,7 @@ func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes [ } status.Pending = append(status.Pending, *pendingRef) } - status.Budget = uint32(newNodesBudget) + status.Budget = newNodesBudget return status } @@ -936,10 +959,35 @@ type kubernetesServerVersionGetter interface { ServerVersion() (*version.Info, error) } +type scalingGroupByID map[string]updatev1alpha1.ScalingGroup + +// getScalingGroupIDsSorted returns the group IDs of all scaling groups with +// scaling groups with the role "control-plane" first. +func (s scalingGroupByID) getScalingGroupIDsSorted() []string { + var controlPlaneGroupIDs, otherGroupIDs []string + for id := range s { + if s[id].Spec.Role == updatev1alpha1.ControlPlaneRole { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, id) + } else { + otherGroupIDs = append(otherGroupIDs, id) + } + } + return append(controlPlaneGroupIDs, otherGroupIDs...) +} + +func (s scalingGroupByID) put(scalingGroup updatev1alpha1.ScalingGroup) { + s[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup +} + +func (s scalingGroupByID) get(scalingGroupID string) (scalingGroup updatev1alpha1.ScalingGroup, ok bool) { + scalingGroup, ok = s[strings.ToLower(scalingGroupID)] + return +} + type newNodeConfig struct { desiredNodeVersion updatev1alpha1.NodeVersion - outdatedNodes []corev1.Node + groups nodeGroups pendingNodes []updatev1alpha1.PendingNode - scalingGroupByID map[string]updatev1alpha1.ScalingGroup - newNodesBudget int + scalingGroupByID scalingGroupByID + newNodesBudget uint32 } diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go index 7fa62ce3a1..1e1353bc67 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_env_test.go @@ -27,6 +27,7 @@ import ( nodemaintenancev1beta1 "github.com/edgelesssys/constellation/v2/3rdparty/node-maintenance-operator/api/v1beta1" mainconstants "github.com/edgelesssys/constellation/v2/internal/constants" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" ) var _ = Describe("NodeVersion controller", func() { @@ -38,10 +39,15 @@ var _ = Describe("NodeVersion controller", func() { // Define utility constants for object names and testing timeouts/durations and intervals. const ( - nodeVersionResourceName = "nodeversion" - firstNodeName = "node-1" - secondNodeName = "node-2" - scalingGroupID = "scaling-group" + nodeVersionResourceName = "nodeversion" + firstWorkerNodeName = "worker-node-1" + secondWorkerNodeName = "worker-node-2" + thirdWorkerNodeName = "worker-node-3" + firstControlPlaneNodeName = "control-plane-node-1" + secondControlPlaneNodeName = "control-plane-node-2" + thirdControlPlaneNodeName = "control-plane-node-3" + scalingGroupIDWorker = "scaling-group-worker" + scalingGroupIDControlPlane = "scaling-group-control-plane" timeout = time.Second * 20 duration = time.Second * 2 @@ -52,17 +58,17 @@ var _ = Describe("NodeVersion controller", func() { KubernetesComponentsReference: "ref-1", } - firstNodeLookupKey := types.NamespacedName{Name: firstNodeName} - secondNodeLookupKey := types.NamespacedName{Name: secondNodeName} + firstNodeLookupKeyWorker := types.NamespacedName{Name: firstWorkerNodeName} + secondNodeLookupKeyWorker := types.NamespacedName{Name: secondWorkerNodeName} + firstNodeLookupKeyControlPlane := types.NamespacedName{Name: firstControlPlaneNodeName} + secondNodeLookupKeyControlPlane := types.NamespacedName{Name: secondControlPlaneNodeName} nodeVersionLookupKey := types.NamespacedName{Name: nodeVersionResourceName} - scalingGroupLookupKey := types.NamespacedName{Name: scalingGroupID} - joiningPendingNodeLookupKey := types.NamespacedName{Name: secondNodeName} - nodeMaintenanceLookupKey := types.NamespacedName{Name: firstNodeName} + scalingGroupLookupKeyWorker := types.NamespacedName{Name: scalingGroupIDWorker} + scalingGroupLookupKeyControlPlane := types.NamespacedName{Name: scalingGroupIDControlPlane} Context("When updating the cluster-wide node version", func() { - testNodeVersionUpdate := func(newNodeVersionSpec updatev1alpha1.NodeVersionSpec) { + testNodeVersionUpdate := func(newNodeVersionSpec updatev1alpha1.NodeVersionSpec, workersFirst bool) { By("creating a node version resource specifying the first node version") - Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersionSpec.ImageReference)).Should(Succeed()) nodeVersion := &updatev1alpha1.NodeVersion{ TypeMeta: metav1.TypeMeta{ APIVersion: "update.edgeless.systems/v1alpha1", @@ -75,16 +81,48 @@ var _ = Describe("NodeVersion controller", func() { } Expect(k8sClient.Create(ctx, nodeVersion)).Should(Succeed()) - By("creating a node resource using the first node image") - fakes.nodeReplacer.setNodeImage(firstNodeName, firstVersionSpec.ImageReference) - fakes.nodeReplacer.setScalingGroupID(firstNodeName, scalingGroupID) - firstNode := &corev1.Node{ + By("creating a control plane node resource using the first node image") + fakes.nodeReplacer.setNodeImage(firstControlPlaneNodeName, firstVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(firstControlPlaneNodeName, scalingGroupIDControlPlane) + firstControlPlaneNode := &corev1.Node{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Node", }, ObjectMeta: metav1.ObjectMeta{ - Name: firstNodeName, + Name: firstControlPlaneNodeName, + Labels: map[string]string{ + "custom-node-label": "custom-node-label-value", + constants.ControlPlaneRoleLabel: "", + }, + Annotations: map[string]string{ + mainconstants.NodeKubernetesComponentsAnnotationKey: firstVersionSpec.KubernetesComponentsReference, + }, + }, + Spec: corev1.NodeSpec{ + ProviderID: firstControlPlaneNodeName, + }, + Status: corev1.NodeStatus{ + Addresses: []corev1.NodeAddress{ + { + Type: corev1.NodeInternalIP, + Address: "192.0.2.1", + }, + }, + }, + } + Expect(k8sClient.Create(ctx, firstControlPlaneNode)).Should(Succeed()) + + By("creating a worker node resource using the first node image") + fakes.nodeReplacer.setNodeImage(firstWorkerNodeName, firstVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(firstWorkerNodeName, scalingGroupIDWorker) + firstWorkerNode := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: firstWorkerNodeName, Labels: map[string]string{ "custom-node-label": "custom-node-label-value", }, @@ -93,24 +131,40 @@ var _ = Describe("NodeVersion controller", func() { }, }, Spec: corev1.NodeSpec{ - ProviderID: firstNodeName, + ProviderID: firstWorkerNodeName, }, } - Expect(k8sClient.Create(ctx, firstNode)).Should(Succeed()) + Expect(k8sClient.Create(ctx, firstWorkerNode)).Should(Succeed()) - By("creating a scaling group resource using the first node image") - Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersionSpec.ImageReference)).Should(Succeed()) - scalingGroup := &updatev1alpha1.ScalingGroup{ + By("creating worker scaling group resource using the first node image") + Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupIDWorker, firstVersionSpec.ImageReference)).Should(Succeed()) + scalingGroupWorker := &updatev1alpha1.ScalingGroup{ ObjectMeta: metav1.ObjectMeta{ - Name: scalingGroupID, + Name: scalingGroupIDWorker, }, Spec: updatev1alpha1.ScalingGroupSpec{ NodeVersion: nodeVersionResourceName, - GroupID: scalingGroupID, + GroupID: scalingGroupIDWorker, Autoscaling: true, + Role: updatev1alpha1.WorkerRole, }, } - Expect(k8sClient.Create(ctx, scalingGroup)).Should(Succeed()) + Expect(k8sClient.Create(ctx, scalingGroupWorker)).Should(Succeed()) + + By("creating control plane scaling group resource using the first node image") + Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupIDControlPlane, firstVersionSpec.ImageReference)).Should(Succeed()) + scalingGroupControlPlane := &updatev1alpha1.ScalingGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: scalingGroupIDControlPlane, + }, + Spec: updatev1alpha1.ScalingGroupSpec{ + NodeVersion: nodeVersionResourceName, + GroupID: scalingGroupIDControlPlane, + Autoscaling: true, + Role: updatev1alpha1.ControlPlaneRole, + }, + } + Expect(k8sClient.Create(ctx, scalingGroupControlPlane)).Should(Succeed()) By("creating a cluster-autoscaler deployment") ctx := context.Background() @@ -167,11 +221,20 @@ var _ = Describe("NodeVersion controller", func() { return 0 } return len(nodeVersion.Status.UpToDate) - }, timeout, interval).Should(Equal(1)) + }, timeout, interval).Should(Equal(2)) By("updating the node version") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateReady) - fakes.nodeReplacer.setCreatedNode(secondNodeName, secondNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDControlPlane, secondControlPlaneNodeName, secondControlPlaneNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDWorker, secondWorkerNodeName, secondWorkerNodeName, nil) + fakes.nodeStateGetter.setNodeState(secondControlPlaneNodeName, updatev1alpha1.NodeStateReady) + fakes.nodeStateGetter.setNodeState(secondWorkerNodeName, updatev1alpha1.NodeStateReady) + // When the pending node CR that the nodeversion loop creates is not detected on the second iteration e.g., because of delay in the KubeAPI + // it creates a second node via the cloud provider api. This is fine because the pending node/mint node is not matched if it's not needed + // and the nodeversion loop will clean up the mint node. + fakes.nodeStateGetter.setNodeState(thirdControlPlaneNodeName, updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(thirdWorkerNodeName, updatev1alpha1.NodeStateCreating) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDControlPlane, thirdControlPlaneNodeName, thirdControlPlaneNodeName, nil) + fakes.nodeReplacer.addCreatedNode(scalingGroupIDWorker, thirdWorkerNodeName, thirdWorkerNodeName, nil) // Eventually the node version with the new NodeVersion spec. Eventually(func() error { if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { @@ -181,172 +244,216 @@ var _ = Describe("NodeVersion controller", func() { return k8sClient.Update(ctx, nodeVersion) }, timeout, interval).Should(Succeed()) - By("checking that there is an outdated node in the status") + By("checking that there are 2 outdated node in the status") Eventually(func() int { if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { return 0 } return len(nodeVersion.Status.Outdated) - }, timeout, interval).Should(Equal(1), "outdated nodes should be 1") + }, timeout, interval).Should(Equal(2), "outdated nodes should be 2") - By("checking that the scaling group is up to date") + By("checking that the control plane scaling group is up to date") Eventually(func() string { - if err := k8sClient.Get(ctx, scalingGroupLookupKey, scalingGroup); err != nil { + if err := k8sClient.Get(ctx, scalingGroupLookupKeyControlPlane, scalingGroupControlPlane); err != nil { return "" } - return scalingGroup.Status.ImageReference + return scalingGroupControlPlane.Status.ImageReference }, timeout, interval).Should(Equal(newNodeVersionSpec.ImageReference)) - By("checking that a pending node is created") - pendingNode := &updatev1alpha1.PendingNode{} - Eventually(func() error { - return k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode) - }, timeout, interval).Should(Succeed()) - Eventually(func() updatev1alpha1.CSPNodeState { - _ = k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode) - return pendingNode.Status.CSPNodeState - }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateReady)) - Eventually(func() int { - if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { - return 0 - } - return len(nodeVersion.Status.Pending) - }, timeout, interval).Should(Equal(1)) - - By("creating a new node resource using the image from the new node version") - fakes.nodeReplacer.setNodeImage(secondNodeName, newNodeVersionSpec.ImageReference) - fakes.nodeReplacer.setScalingGroupID(secondNodeName, scalingGroupID) - secondNode := &corev1.Node{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secondNodeName, - }, - Spec: corev1.NodeSpec{ - ProviderID: secondNodeName, - }, - } - Expect(k8sClient.Create(ctx, secondNode)).Should(Succeed()) - - By("marking the new node as AwaitingAnnotation") - Eventually(func() int { - err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) - if err != nil { - return 0 - } - return len(nodeVersion.Status.AwaitingAnnotation) - }, timeout, interval).Should(Equal(1)) - // add a JoiningNode CR for the new node - joiningNode := &updatev1alpha1.JoiningNode{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "update.edgeless.systems/v1alpha1", - Kind: "JoiningNode", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: secondNodeName, - }, - Spec: updatev1alpha1.JoiningNodeSpec{ - Name: secondNodeName, - ComponentsReference: newNodeVersionSpec.KubernetesComponentsReference, - }, - } - Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) - Eventually(func() int { - err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) - if err != nil { - return 1 - } - return len(nodeVersion.Status.AwaitingAnnotation) - }, timeout, interval).Should(Equal(0)) - - By("checking that the new node is properly annotated") - Eventually(func() error { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return err - } - // check nodeImageAnnotation annotation - if _, ok := secondNode.Annotations[nodeImageAnnotation]; !ok { - return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, nodeImageAnnotation) - } - // check mainconstants.NodeKubernetesComponentsAnnotationKey annotation - if _, ok := secondNode.Annotations[mainconstants.NodeKubernetesComponentsAnnotationKey]; !ok { - return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, mainconstants.NodeKubernetesComponentsAnnotationKey) - } - return nil - }, timeout, interval).Should(Succeed()) - - By("checking that the nodes are paired as donor and heir") - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { - return nil - } - return firstNode.Annotations - }, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName)) - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return nil - } - return secondNode.Annotations - }, timeout, interval).Should(HaveKeyWithValue(donorAnnotation, firstNodeName)) - - Eventually(func() error { - if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { - return err - } - if len(nodeVersion.Status.Donors) != 1 { - return fmt.Errorf("node version %s has %d donors, expected 1", nodeVersion.Name, len(nodeVersion.Status.Donors)) - } - if len(nodeVersion.Status.Heirs) != 1 { - return fmt.Errorf("node version %s has %d heirs, expected 1", nodeVersion.Name, len(nodeVersion.Status.Heirs)) + By("checking that the worker scaling group is up to date") + Eventually(func() string { + if err := k8sClient.Get(ctx, scalingGroupLookupKeyWorker, scalingGroupWorker); err != nil { + return "" } - return nil - }, timeout, interval).Should(Succeed()) - Expect(k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode)).Should(Not(Succeed())) + return scalingGroupWorker.Status.ImageReference + }, timeout, interval).Should(Equal(newNodeVersionSpec.ImageReference)) - By("checking that node labels are copied to the heir") - Eventually(func() map[string]string { - if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + replaceNodeTest := func(firstNodeLookupKey, secondNodeLookupKey types.NamespacedName, firstNodeName, secondNodeName, scalingGroupID string, firstNode *corev1.Node, expectPendingNode bool) *corev1.Node { + By("checking that the pending node is created") + pendingNode := &updatev1alpha1.PendingNode{} + // If we try to upgrade the worker nodes first, we expect here that the worker node is not created + if !expectPendingNode { + Consistently(func() bool { + if err := k8sClient.Get(ctx, secondNodeLookupKey, pendingNode); err != nil { + return false + } + return true + }, duration, interval).Should(BeFalse()) return nil } - return firstNode.Labels - }, timeout, interval).Should(HaveKeyWithValue("custom-node-label", "custom-node-label-value")) - - By("marking the new node as ready") - Eventually(func() error { - if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { - return err + Eventually(func() bool { + if err := k8sClient.Get(ctx, secondNodeLookupKey, pendingNode); err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + Eventually(func() updatev1alpha1.CSPNodeState { + _ = k8sClient.Get(ctx, secondNodeLookupKey, pendingNode) + return pendingNode.Status.CSPNodeState + }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateReady)) + Eventually(func() bool { + if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { + return false + } + return len(nodeVersion.Status.Pending) >= 1 + }, timeout, interval).Should(BeTrue()) + + By("creating a new node resource using the image from the new node version") + fakes.nodeReplacer.setNodeImage(secondNodeName, newNodeVersionSpec.ImageReference) + fakes.nodeReplacer.setScalingGroupID(secondNodeName, scalingGroupID) + var labels map[string]string + if _, ok := firstNode.Labels[constants.ControlPlaneRoleLabel]; ok { + labels = map[string]string{ + constants.ControlPlaneRoleLabel: "", + } } - secondNode.Status.Conditions = []corev1.NodeCondition{ - { - Type: corev1.NodeReady, - Status: corev1.ConditionTrue, + secondNode := &corev1.Node{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secondNodeName, + Labels: labels, + }, + Spec: corev1.NodeSpec{ + ProviderID: secondNodeName, }, } - return k8sClient.Status().Update(ctx, secondNode) - }, timeout, interval).Should(Succeed()) - - By("waiting for a NodeMaintenance resource to be created") - nodeMaintenance := &nodemaintenancev1beta1.NodeMaintenance{} - Eventually(func() error { - return k8sClient.Get(ctx, nodeMaintenanceLookupKey, nodeMaintenance) - }, timeout, interval).Should(Succeed()) - - By("marking the NodeMaintenance as successful") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateTerminated) - Eventually(func() error { - if err := k8sClient.Get(ctx, nodeMaintenanceLookupKey, nodeMaintenance); err != nil { - return err + Expect(k8sClient.Create(ctx, secondNode)).Should(Succeed()) + + By("marking the new node as AwaitingAnnotation") + Eventually(func() int { + err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) + if err != nil { + return 0 + } + return len(nodeVersion.Status.AwaitingAnnotation) + }, timeout, interval).Should(Equal(1)) + // add a JoiningNode CR for the new node + joiningNode := &updatev1alpha1.JoiningNode{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "update.edgeless.systems/v1alpha1", + Kind: "JoiningNode", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: secondNodeName, + }, + Spec: updatev1alpha1.JoiningNodeSpec{ + Name: secondNodeName, + ComponentsReference: newNodeVersionSpec.KubernetesComponentsReference, + }, } - nodeMaintenance.Status.Phase = nodemaintenancev1beta1.MaintenanceSucceeded - return k8sClient.Status().Update(ctx, nodeMaintenance) - }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed()) + Eventually(func() int { + err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion) + if err != nil { + return 1 + } + return len(nodeVersion.Status.AwaitingAnnotation) + }, timeout, interval).Should(Equal(0)) + + By("checking that the new node is properly annotated") + Eventually(func() error { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return err + } + // check nodeImageAnnotation annotation + if _, ok := secondNode.Annotations[nodeImageAnnotation]; !ok { + return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, nodeImageAnnotation) + } + // check mainconstants.NodeKubernetesComponentsAnnotationKey annotation + if _, ok := secondNode.Annotations[mainconstants.NodeKubernetesComponentsAnnotationKey]; !ok { + return fmt.Errorf("node %s is missing %s annotation", secondNode.Name, mainconstants.NodeKubernetesComponentsAnnotationKey) + } + return nil + }, timeout, interval).Should(Succeed()) + + By("checking that the nodes are paired as donor and heir") + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + return nil + } + return firstNode.Annotations + }, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName)) + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return nil + } + return secondNode.Annotations + }, timeout, interval).Should(HaveKeyWithValue(donorAnnotation, firstNodeName)) + + Eventually(func() error { + if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil { + return err + } + if len(nodeVersion.Status.Donors) != 1 { + return fmt.Errorf("node version %s has %d donors, expected 1", nodeVersion.Name, len(nodeVersion.Status.Donors)) + } + if len(nodeVersion.Status.Heirs) != 1 { + return fmt.Errorf("node version %s has %d heirs, expected 1", nodeVersion.Name, len(nodeVersion.Status.Heirs)) + } + return nil + }, timeout, interval).Should(Succeed()) + Expect(k8sClient.Get(ctx, secondNodeLookupKey, pendingNode)).Should(Not(Succeed())) + + By("checking that node labels are copied to the heir") + Eventually(func() map[string]string { + if err := k8sClient.Get(ctx, firstNodeLookupKey, firstNode); err != nil { + return nil + } + return firstNode.Labels + }, timeout, interval).Should(HaveKeyWithValue("custom-node-label", "custom-node-label-value")) + + By("marking the new node as ready") + Eventually(func() error { + if err := k8sClient.Get(ctx, secondNodeLookupKey, secondNode); err != nil { + return err + } + secondNode.Status.Conditions = []corev1.NodeCondition{ + { + Type: corev1.NodeReady, + Status: corev1.ConditionTrue, + }, + } + return k8sClient.Status().Update(ctx, secondNode) + }, timeout, interval).Should(Succeed()) + + By("waiting for a NodeMaintenance resource to be created") + nodeMaintenance := &nodemaintenancev1beta1.NodeMaintenance{} + Eventually(func() error { + return k8sClient.Get(ctx, firstNodeLookupKey, nodeMaintenance) + }, timeout, interval).Should(Succeed()) + + By("marking the NodeMaintenance as successful") + fakes.nodeStateGetter.setNodeState(firstNodeName, updatev1alpha1.NodeStateTerminated) + Eventually(func() error { + if err := k8sClient.Get(ctx, firstNodeLookupKey, nodeMaintenance); err != nil { + return err + } + nodeMaintenance.Status.Phase = nodemaintenancev1beta1.MaintenanceSucceeded + return k8sClient.Status().Update(ctx, nodeMaintenance) + }, timeout, interval).Should(Succeed()) + + By("checking that the outdated node is removed") + Eventually(func() error { + return k8sClient.Get(ctx, firstNodeLookupKey, firstNode) + }, timeout, interval).Should(Not(Succeed())) + + return secondNode + } - By("checking that the outdated node is removed") - Eventually(func() error { - return k8sClient.Get(ctx, firstNodeLookupKey, firstNode) - }, timeout, interval).Should(Not(Succeed())) + var createdControlPlane *corev1.Node + var createdWorkerNode *corev1.Node + if workersFirst { + _ = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, false) + createdControlPlane = replaceNodeTest(firstNodeLookupKeyControlPlane, secondNodeLookupKeyControlPlane, firstControlPlaneNodeName, secondControlPlaneNodeName, scalingGroupIDControlPlane, firstControlPlaneNode, true) + createdWorkerNode = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, true) + } else { + createdControlPlane = replaceNodeTest(firstNodeLookupKeyControlPlane, secondNodeLookupKeyControlPlane, firstControlPlaneNodeName, secondControlPlaneNodeName, scalingGroupIDControlPlane, firstControlPlaneNode, true) + createdWorkerNode = replaceNodeTest(firstNodeLookupKeyWorker, secondNodeLookupKeyWorker, firstWorkerNodeName, secondWorkerNodeName, scalingGroupIDWorker, firstWorkerNode, true) + } By("checking that all nodes are up-to-date") Eventually(func() int { @@ -355,29 +462,48 @@ var _ = Describe("NodeVersion controller", func() { return 0 } return len(nodeVersion.Status.UpToDate) - }, timeout, interval).Should(Equal(1)) + }, timeout, interval).Should(Equal(2)) By("cleaning up all resources") Expect(k8sClient.Delete(ctx, nodeVersion)).Should(Succeed()) - Expect(k8sClient.Delete(ctx, scalingGroup)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, scalingGroupWorker)).Should(Succeed()) Expect(k8sClient.Delete(ctx, autoscalerDeployment)).Should(Succeed()) Expect(k8sClient.Delete(ctx, strategy)).Should(Succeed()) - Expect(k8sClient.Delete(ctx, secondNode)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, createdControlPlane)).Should(Succeed()) + Expect(k8sClient.Delete(ctx, createdWorkerNode)).Should(Succeed()) } When("Updating the image reference", func() { It("Should update every node in the cluster", func() { - testNodeVersionUpdate(updatev1alpha1.NodeVersionSpec{ - ImageReference: "version-2", - KubernetesComponentsReference: "ref-1", - }) + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-2", + KubernetesComponentsReference: "ref-1", + }, + false, + ) }) }) When("Updating the Kubernetes components reference", func() { It("Should update every node in the cluster", func() { - testNodeVersionUpdate(updatev1alpha1.NodeVersionSpec{ - ImageReference: "version-1", - KubernetesComponentsReference: "ref-2", - }) + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-1", + KubernetesComponentsReference: "ref-2", + }, + false, + ) + }) + }) + When("Updating the Kubernetes components reference and wanting to upgrade the worker nodes first", func() { + It("should fail to update the worker nodes before the control plane nodes", func() { + testNodeVersionUpdate( + updatev1alpha1.NodeVersionSpec{ + ImageReference: "version-1", + KubernetesComponentsReference: "ref-2", + MaxNodeBudget: 2, + }, + true, + ) }) }) }) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index c9ae880421..4f926b978a 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -9,6 +9,7 @@ package controllers import ( "context" "errors" + "strconv" "sync" "testing" @@ -21,6 +22,7 @@ import ( mainconstants "github.com/edgelesssys/constellation/v2/internal/constants" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" ) func TestAnnotateNodes(t *testing.T) { @@ -332,7 +334,7 @@ func TestCreateNewNodes(t *testing.T) { outdatedNodes []corev1.Node pendingNodes []updatev1alpha1.PendingNode scalingGroupByID map[string]updatev1alpha1.ScalingGroup - budget int + budget uint32 wantCreateCalls []string }{ "no outdated nodes": { @@ -560,6 +562,51 @@ func TestCreateNewNodes(t *testing.T) { budget: 2, wantCreateCalls: []string{"scaling-group"}, }, + "outdated nodes still contain control plane nodes": { + outdatedNodes: []corev1.Node{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "control-plane-node", + Annotations: map[string]string{ + scalingGroupAnnotation: "control-plane-scaling-group", + }, + Labels: map[string]string{ + constants.ControlPlaneRoleLabel: "", + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Name: "node", + Annotations: map[string]string{ + scalingGroupAnnotation: "scaling-group", + }, + }, + }, + }, + scalingGroupByID: map[string]updatev1alpha1.ScalingGroup{ + "scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "scaling-group", + Role: updatev1alpha1.WorkerRole, + }, + Status: updatev1alpha1.ScalingGroupStatus{ + ImageReference: "image", + }, + }, + "control-plane-scaling-group": { + Spec: updatev1alpha1.ScalingGroupSpec{ + GroupID: "control-plane-scaling-group", + Role: updatev1alpha1.ControlPlaneRole, + }, + Status: updatev1alpha1.ScalingGroupStatus{ + ImageReference: "image", + }, + }, + }, + budget: 2, + wantCreateCalls: []string{"control-plane-scaling-group"}, + }, "scaling group does not exist": { outdatedNodes: []corev1.Node{ { @@ -592,7 +639,10 @@ func TestCreateNewNodes(t *testing.T) { }, Scheme: getScheme(t), } - newNodeConfig := newNodeConfig{desiredNodeImage, tc.outdatedNodes, tc.pendingNodes, tc.scalingGroupByID, tc.budget} + groups := nodeGroups{ + Outdated: tc.outdatedNodes, + } + newNodeConfig := newNodeConfig{desiredNodeImage, groups, tc.pendingNodes, tc.scalingGroupByID, tc.budget} err := reconciler.createNewNodes(context.Background(), newNodeConfig) require.NoError(err) assert.Equal(tc.wantCreateCalls, reconciler.nodeReplacer.(*stubNodeReplacerWriter).createCalls) @@ -752,16 +802,22 @@ func TestGroupNodes(t *testing.T) { assert.Equal(wantNodeGroups, groups) } +// stubNode contains all information usually associated with a node freshly +// created by the cloud provider. +type stubNode struct { + name string + providerID string +} + type stubNodeReplacer struct { sync.RWMutex - nodeImages map[string]string - scalingGroups map[string]string - createNodeName string - createProviderID string - nodeImageErr error - scalingGroupIDErr error - createErr error - deleteErr error + nodeImages map[string]string + scalingGroups map[string]string + createNodesByScalingGroupID map[string][]stubNode + nodeImageErr error + scalingGroupIDErr error + createErr error + deleteErr error } func (r *stubNodeReplacer) GetNodeImage(_ context.Context, providerID string) (string, error) { @@ -776,10 +832,25 @@ func (r *stubNodeReplacer) GetScalingGroupID(_ context.Context, providerID strin return r.scalingGroups[providerID], r.scalingGroupIDErr } -func (r *stubNodeReplacer) CreateNode(_ context.Context, _ string) (nodeName, providerID string, err error) { +// CreateNode stubs the cloud provider API call to create a node. +func (r *stubNodeReplacer) CreateNode(_ context.Context, scalingGroupID string) (nodeName, providerID string, err error) { r.RLock() defer r.RUnlock() - return r.createNodeName, r.createProviderID, r.createErr + nodes, ok := r.createNodesByScalingGroupID[scalingGroupID] + if !ok { + panic("unexpected call to CreateNode with scaling group ID " + scalingGroupID + ", existing scaling group IDs: " + strconv.Itoa(len(r.createNodesByScalingGroupID))) + } + if len(nodes) == 0 { + panic("unexpected call to CreateNode with scaling group ID " + scalingGroupID + ", no nodes left") + } + + nodeName = nodes[0].name + providerID = nodes[0].providerID + err = r.createErr + + r.createNodesByScalingGroupID[scalingGroupID] = nodes[1:] + + return } func (r *stubNodeReplacer) DeleteNode(_ context.Context, _ string) error { @@ -808,12 +879,14 @@ func (r *stubNodeReplacer) setScalingGroupID(providerID, scalingGroupID string) r.scalingGroups[providerID] = scalingGroupID } -func (r *stubNodeReplacer) setCreatedNode(nodeName, providerID string, err error) { +func (r *stubNodeReplacer) addCreatedNode(scalingGroupID, nodeName, providerID string, _ error) { r.Lock() defer r.Unlock() - r.createNodeName = nodeName - r.createProviderID = providerID - r.createErr = err + if r.createNodesByScalingGroupID == nil { + r.createNodesByScalingGroupID = make(map[string][]stubNode) + } + r.createNodesByScalingGroupID[scalingGroupID] = append(r.createNodesByScalingGroupID[scalingGroupID], stubNode{nodeName, providerID}) + r.createErr = nil } func (r *stubNodeReplacer) reset() { @@ -821,8 +894,7 @@ func (r *stubNodeReplacer) reset() { defer r.Unlock() r.nodeImages = nil r.scalingGroups = nil - r.createNodeName = "" - r.createProviderID = "" + r.createNodesByScalingGroupID = nil r.createErr = nil r.deleteErr = nil } @@ -891,3 +963,11 @@ func (*unimplementedNodeReplacer) CreateNode(_ context.Context, _ string) (nodeN func (*unimplementedNodeReplacer) DeleteNode(_ context.Context, _ string) error { panic("unimplemented") } + +type stubEtcdRemover struct { + deleteErr error +} + +func (r *stubEtcdRemover) RemoveEtcdMemberFromCluster(_ context.Context, _ string) error { + return r.deleteErr +} diff --git a/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go b/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go index 05e5de4eee..d9a6416d06 100644 --- a/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go +++ b/operators/constellation-node-operator/controllers/pendingnode_controller_env_test.go @@ -64,7 +64,7 @@ var _ = Describe("PendingNode controller", func() { Context("When creating pending node with goal join", func() { It("Should terminate the node after failing to join by the deadline", func() { By("setting the CSP node state to creating") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateCreating) By("creating a pending node resource") ctx := context.Background() @@ -77,7 +77,7 @@ var _ = Describe("PendingNode controller", func() { Name: pendingNodeName, }, Spec: updatev1alpha1.PendingNodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, ScalingGroupID: "scaling-group-id", NodeName: "test-node", Goal: updatev1alpha1.NodeGoalJoin, @@ -120,7 +120,7 @@ var _ = Describe("PendingNode controller", func() { }, timeout, interval).Should(Equal(updatev1alpha1.NodeGoalLeave)) By("setting the CSP node state to terminated") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateTerminated) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateTerminated) // trigger reconciliation before regular check interval to speed up test by changing the spec Eventually(func() error { if err := k8sClient.Get(ctx, pendingNodeLookupKey, pendingNode); err != nil { @@ -138,7 +138,7 @@ var _ = Describe("PendingNode controller", func() { It("Should should detect successful node join", func() { By("setting the CSP node state to creating") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateCreating) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateCreating) By("creating a pending node resource") ctx := context.Background() @@ -151,7 +151,7 @@ var _ = Describe("PendingNode controller", func() { Name: pendingNodeName, }, Spec: updatev1alpha1.PendingNodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, ScalingGroupID: "scaling-group-id", NodeName: "test-node", Goal: updatev1alpha1.NodeGoalJoin, @@ -178,7 +178,7 @@ var _ = Describe("PendingNode controller", func() { }, timeout, interval).Should(Equal(updatev1alpha1.NodeStateCreating)) By("setting the CSP node state to ready") - fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateReady) + fakes.nodeStateGetter.setNodeState(pendingNodeName, updatev1alpha1.NodeStateReady) By("creating a new node resource with the same node name and provider ID") node := &corev1.Node{ @@ -190,7 +190,7 @@ var _ = Describe("PendingNode controller", func() { Name: "test-node", }, Spec: corev1.NodeSpec{ - ProviderID: "provider-id", + ProviderID: pendingNodeName, }, } Expect(k8sClient.Create(ctx, node)).Should(Succeed()) diff --git a/operators/constellation-node-operator/controllers/pendingnode_controller_test.go b/operators/constellation-node-operator/controllers/pendingnode_controller_test.go index 1a564af768..d5ca55fe6d 100644 --- a/operators/constellation-node-operator/controllers/pendingnode_controller_test.go +++ b/operators/constellation-node-operator/controllers/pendingnode_controller_test.go @@ -231,15 +231,20 @@ func TestReachedGoal(t *testing.T) { type stubNodeStateGetter struct { sync.RWMutex - nodeState updatev1alpha1.CSPNodeState + nodeStates map[string]updatev1alpha1.CSPNodeState nodeStateErr error deleteNodeErr error } -func (g *stubNodeStateGetter) GetNodeState(_ context.Context, _ string) (updatev1alpha1.CSPNodeState, error) { +func (g *stubNodeStateGetter) GetNodeState(_ context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) { g.RLock() defer g.RUnlock() - return g.nodeState, g.nodeStateErr + + if _, ok := g.nodeStates[providerID]; !ok { + panic("unexpected call to GetNodeState") + } + + return g.nodeStates[providerID], g.nodeStateErr } func (g *stubNodeStateGetter) DeleteNode(_ context.Context, _ string) error { @@ -250,8 +255,19 @@ func (g *stubNodeStateGetter) DeleteNode(_ context.Context, _ string) error { // thread safe methods to update the stub while in use -func (g *stubNodeStateGetter) setNodeState(nodeState updatev1alpha1.CSPNodeState) { +func (g *stubNodeStateGetter) setNodeState(providerID string, nodeState updatev1alpha1.CSPNodeState) { + g.Lock() + defer g.Unlock() + if g.nodeStates == nil { + g.nodeStates = make(map[string]updatev1alpha1.CSPNodeState) + } + g.nodeStates[providerID] = nodeState +} + +func (g *stubNodeStateGetter) reset() { g.Lock() defer g.Unlock() - g.nodeState = nodeState + g.nodeStates = nil + g.nodeStateErr = nil + g.deleteNodeErr = nil } diff --git a/operators/constellation-node-operator/controllers/scalinggroup_controller.go b/operators/constellation-node-operator/controllers/scalinggroup_controller.go index 7336467c0a..35743f3cc7 100644 --- a/operators/constellation-node-operator/controllers/scalinggroup_controller.go +++ b/operators/constellation-node-operator/controllers/scalinggroup_controller.go @@ -10,6 +10,7 @@ import ( "context" "strings" + k8sErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" @@ -92,8 +93,10 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request } meta.SetStatusCondition(&desiredScalingGroup.Status.Conditions, outdatedCondition) if err := r.Status().Update(ctx, &desiredScalingGroup); err != nil { - logr.Error(err, "Unable to update AutoscalingStrategy status") - return ctrl.Result{}, err + if !k8sErrors.IsConflict(err) { + logr.Error(err, "Unable to update Scaling Group status") + } + return ctrl.Result{Requeue: true}, err } if !imagesMatch { diff --git a/operators/constellation-node-operator/controllers/suite_test.go b/operators/constellation-node-operator/controllers/suite_test.go index dd2df34078..eafc162814 100644 --- a/operators/constellation-node-operator/controllers/suite_test.go +++ b/operators/constellation-node-operator/controllers/suite_test.go @@ -123,6 +123,7 @@ var _ = BeforeSuite(func() { err = (&NodeVersionReconciler{ kubernetesServerVersionGetter: fakes.k8sVerGetter, nodeReplacer: fakes.nodeReplacer, + etcdRemover: fakes.etcdRemover, Client: k8sManager.GetClient(), Scheme: k8sManager.GetScheme(), }).SetupWithManager(k8sManager) @@ -147,12 +148,13 @@ type fakeCollection struct { nodeStateGetter *stubNodeStateGetter nodeReplacer *stubNodeReplacer k8sVerGetter *stubKubernetesServerVersionGetter + etcdRemover *stubEtcdRemover clock *testclock.FakeClock } func (c *fakeCollection) reset() { c.scalingGroupUpdater.reset() - c.nodeStateGetter.setNodeState("") + c.nodeStateGetter.reset() c.nodeReplacer.reset() } @@ -162,6 +164,7 @@ func newFakes() fakeCollection { nodeStateGetter: &stubNodeStateGetter{}, nodeReplacer: &stubNodeReplacer{}, k8sVerGetter: &stubKubernetesServerVersionGetter{}, + etcdRemover: &stubEtcdRemover{}, clock: testclock.NewFakeClock(time.Now()), } } diff --git a/operators/constellation-node-operator/internal/constants/constants.go b/operators/constellation-node-operator/internal/constants/constants.go index b057df59cb..db49353003 100644 --- a/operators/constellation-node-operator/internal/constants/constants.go +++ b/operators/constellation-node-operator/internal/constants/constants.go @@ -19,4 +19,6 @@ const ( PlaceholderControlPlaneScalingGroupName = "control-planes-id" // PlaceholderWorkerScalingGroupName name of the worker scaling group used if upgrades are not yet supported. PlaceholderWorkerScalingGroupName = "workers-id" + // ControlPlaneRoleLabel label used to identify control plane nodes. + ControlPlaneRoleLabel = "node-role.kubernetes.io/control-plane" ) diff --git a/operators/constellation-node-operator/internal/controlplane/BUILD.bazel b/operators/constellation-node-operator/internal/controlplane/BUILD.bazel index 65651541d0..c2495adbb4 100644 --- a/operators/constellation-node-operator/internal/controlplane/BUILD.bazel +++ b/operators/constellation-node-operator/internal/controlplane/BUILD.bazel @@ -18,6 +18,7 @@ go_test( srcs = ["controlplane_test.go"], embed = [":controlplane"], deps = [ + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_k8s_api//core/v1:core", diff --git a/operators/constellation-node-operator/internal/controlplane/controlplane_test.go b/operators/constellation-node-operator/internal/controlplane/controlplane_test.go index a089f02b34..6aba0137bd 100644 --- a/operators/constellation-node-operator/internal/controlplane/controlplane_test.go +++ b/operators/constellation-node-operator/internal/controlplane/controlplane_test.go @@ -11,6 +11,7 @@ import ( "errors" "testing" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -29,7 +30,7 @@ func TestListControlPlaneIPs(t *testing.T) { nodes: []corev1.Node{ { ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{ Type: corev1.NodeInternalIP, diff --git a/operators/constellation-node-operator/internal/etcd/BUILD.bazel b/operators/constellation-node-operator/internal/etcd/BUILD.bazel index f41f5bbf5e..f4e6ced77d 100644 --- a/operators/constellation-node-operator/internal/etcd/BUILD.bazel +++ b/operators/constellation-node-operator/internal/etcd/BUILD.bazel @@ -19,6 +19,7 @@ go_test( srcs = ["etcd_test.go"], embed = [":etcd"], deps = [ + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_etcd_go_etcd_api_v3//etcdserverpb", diff --git a/operators/constellation-node-operator/internal/etcd/etcd_test.go b/operators/constellation-node-operator/internal/etcd/etcd_test.go index 5775140cbc..30e04d817e 100644 --- a/operators/constellation-node-operator/internal/etcd/etcd_test.go +++ b/operators/constellation-node-operator/internal/etcd/etcd_test.go @@ -11,6 +11,7 @@ import ( "errors" "testing" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" pb "go.etcd.io/etcd/api/v3/etcdserverpb" @@ -124,7 +125,7 @@ func TestGetInitialEndpoints(t *testing.T) { nodes: []corev1.Node{ { ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{"node-role.kubernetes.io/control-plane": ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{ Type: corev1.NodeInternalIP, diff --git a/operators/constellation-node-operator/internal/node/BUILD.bazel b/operators/constellation-node-operator/internal/node/BUILD.bazel index 3726946ac1..080257817c 100644 --- a/operators/constellation-node-operator/internal/node/BUILD.bazel +++ b/operators/constellation-node-operator/internal/node/BUILD.bazel @@ -8,6 +8,7 @@ go_library( visibility = ["//operators/constellation-node-operator:__subpackages__"], deps = [ "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@io_k8s_api//core/v1:core", ], ) @@ -18,6 +19,7 @@ go_test( embed = [":node"], deps = [ "//operators/constellation-node-operator/api/v1alpha1", + "//operators/constellation-node-operator/internal/constants", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", "@io_k8s_api//core/v1:core", diff --git a/operators/constellation-node-operator/internal/node/node.go b/operators/constellation-node-operator/internal/node/node.go index 854b49906f..ecfcb61587 100644 --- a/operators/constellation-node-operator/internal/node/node.go +++ b/operators/constellation-node-operator/internal/node/node.go @@ -11,11 +11,10 @@ import ( "regexp" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" corev1 "k8s.io/api/core/v1" ) -const controlPlaneRoleLabel = "node-role.kubernetes.io/control-plane" - var reservedHostRegex = regexp.MustCompile(`^(.+\.|)(kubernetes|k8s)\.io(/.*)?$`) // Ready checks if a kubernetes node has the `NodeReady` condition set to true. @@ -40,7 +39,7 @@ func VPCIP(node *corev1.Node) (string, error) { // IsControlPlaneNode returns true if the node is a control plane node. func IsControlPlaneNode(node *corev1.Node) bool { - _, ok := node.Labels[controlPlaneRoleLabel] + _, ok := node.Labels[constants.ControlPlaneRoleLabel] return ok } diff --git a/operators/constellation-node-operator/internal/node/node_test.go b/operators/constellation-node-operator/internal/node/node_test.go index 8a11a19aa8..34b706b5e3 100644 --- a/operators/constellation-node-operator/internal/node/node_test.go +++ b/operators/constellation-node-operator/internal/node/node_test.go @@ -10,6 +10,7 @@ import ( "testing" updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/api/v1alpha1" + "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/internal/constants" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" @@ -69,7 +70,7 @@ func TestIsControlPlaneNode(t *testing.T) { "node with control-plane role label": { node: corev1.Node{ ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{controlPlaneRoleLabel: ""}, + Labels: map[string]string{constants.ControlPlaneRoleLabel: ""}, }, }, wantControlPlane: true, diff --git a/terraform-provider-constellation/docs/resources/cluster.md b/terraform-provider-constellation/docs/resources/cluster.md index 542200750b..5befe11470 100644 --- a/terraform-provider-constellation/docs/resources/cluster.md +++ b/terraform-provider-constellation/docs/resources/cluster.md @@ -69,7 +69,7 @@ resource "constellation_cluster" "azure_example" { See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview/clouds) that Constellation supports. - `image` (Attributes) Constellation OS Image to use on the nodes. (see [below for nested schema](#nestedatt--image)) - `init_secret` (String) Secret used for initialization of the cluster. -- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.28.13 v1.29.8 v1.30.4]. +- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.29.8 v1.30.4 v1.31.1]. - `master_secret` (String) Hex-encoded 32-byte master secret for the cluster. - `master_secret_salt` (String) Hex-encoded 32-byte master secret salt for the cluster. - `measurement_salt` (String) Hex-encoded 32-byte measurement salt for the cluster.