From 35c4cb12b0935e2b3cf07804cea6851110c715a4 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 7 Oct 2024 23:46:30 +0200 Subject: [PATCH 1/6] versions: add k8s 1.31, remove k8s 1.28 --- .github/workflows/e2e-test-daily.yml | 2 +- .github/workflows/e2e-test-internal-lb.yml | 2 +- .../workflows/e2e-test-marketplace-image.yml | 2 +- .github/workflows/e2e-test-release.yml | 20 +-- .../workflows/e2e-test-terraform-provider.yml | 2 +- .github/workflows/e2e-test-weekly.yml | 30 ++--- .github/workflows/e2e-test.yml | 2 +- docs/docs/architecture/versions.md | 2 +- docs/docs/reference/cli.md | 2 +- internal/constellation/kubecmd/BUILD.bazel | 5 +- internal/constellation/kubecmd/kubecmd.go | 56 ++++---- .../constellation/kubecmd/kubecmd_test.go | 124 ++++++++++++++++++ internal/versions/versions.go | 52 ++++---- .../docs/resources/cluster.md | 2 +- 14 files changed, 212 insertions(+), 91 deletions(-) diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index 5e36add52e..e9ce79e11d 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.28"] # should be default + kubernetesVersion: ["1.29"] # should be default attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] test: ["sonobuoy quick"] diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index cee93f7eb0..0371605894 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index 760df932b1..d0f9e1f533 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index 716c9138d7..84e45ef628 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -73,53 +73,53 @@ jobs: - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-24.04" clusterCreation: "cli" - test: "sonobuoy full" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" runner: "ubuntu-24.04" clusterCreation: "cli" diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 3c465ab973..5054bab186 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -41,7 +41,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.28" + default: "1.29" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 117b2bcce7..3df6bb5a88 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -89,53 +89,53 @@ jobs: - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" - test: "sonobuoy quick" refStream: "ref/main/stream/debug/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.28" + kubernetes-version: "v1.29" clusterCreation: "cli" @@ -290,27 +290,27 @@ jobs: - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-es" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "gcp-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "azure-tdx" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" - test: "verify" refStream: "ref/release/stream/stable/?" attestationVariant: "aws-sev-snp" - kubernetes-version: "v1.29" + kubernetes-version: "v1.30" clusterCreation: "cli" runs-on: ubuntu-24.04 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 30a649817d..9c4c8d0fb2 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -44,7 +44,7 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" + default: "1.30" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/docs/docs/architecture/versions.md b/docs/docs/architecture/versions.md index d9e28bcc70..efb97b9320 100644 --- a/docs/docs/architecture/versions.md +++ b/docs/docs/architecture/versions.md @@ -16,6 +16,6 @@ Subsequent Constellation releases drop support for the oldest (and deprecated) K The following Kubernetes versions are currently supported: -* v1.28.15 * v1.29.13 * v1.30.9 +* v1.31.1 diff --git a/docs/docs/reference/cli.md b/docs/docs/reference/cli.md index 99acef5203..ebddf84c9a 100644 --- a/docs/docs/reference/cli.md +++ b/docs/docs/reference/cli.md @@ -80,7 +80,7 @@ constellation config generate {aws|azure|gcp|openstack|qemu|stackit} [flags] ``` -a, --attestation string attestation variant to use {aws-sev-snp|aws-nitro-tpm|azure-sev-snp|azure-tdx|azure-trustedlaunch|gcp-sev-snp|gcp-sev-es|qemu-vtpm}. If not specified, the default for the cloud provider is used -h, --help help for generate - -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.29") + -k, --kubernetes string Kubernetes version to use in format MAJOR.MINOR (default "v1.30") -t, --tags strings additional tags for created resources given a list of key=value ``` diff --git a/internal/constellation/kubecmd/BUILD.bazel b/internal/constellation/kubecmd/BUILD.bazel index 71bae3c32d..aca26d0bb2 100644 --- a/internal/constellation/kubecmd/BUILD.bazel +++ b/internal/constellation/kubecmd/BUILD.bazel @@ -30,8 +30,11 @@ go_library( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/runtime/serializer/json", "@io_k8s_client_go//util/retry", - "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/scheme", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta4", "@io_k8s_sigs_yaml//:yaml", ], ) diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index 1ebf992654..a193f2528e 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -42,9 +42,11 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + k8sjson "k8s.io/apimachinery/pkg/runtime/serializer/json" "k8s.io/client-go/util/retry" - kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" - "sigs.k8s.io/yaml" + "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm" + kubeadmscheme "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/scheme" + kubeadmv1beta4 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta4" ) // ErrInProgress signals that an upgrade is in progress inside the cluster. @@ -234,9 +236,23 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) - if err != nil { - return fmt.Errorf("getting ClusterConfig: %w", err) + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) } existingSANs := make(map[string]struct{}) @@ -264,7 +280,10 @@ func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNa clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) sort.Strings(clusterConfiguration.APIServer.CertSANs) - newConfigYAML, err := yaml.Marshal(clusterConfiguration) + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) if err != nil { return fmt.Errorf("marshaling ClusterConfiguration: %w", err) } @@ -316,31 +335,6 @@ func (k *KubeCmd) getConstellationVersion(ctx context.Context) (updatev1alpha1.N return nodeVersion, nil } -// getClusterConfiguration fetches the kubeadm-config configmap from the cluster, extracts the config -// and returns both the full configmap and the ClusterConfiguration. -func (k *KubeCmd) getClusterConfiguration(ctx context.Context) (kubeadmv1beta3.ClusterConfiguration, *corev1.ConfigMap, error) { - var existingConf *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - existingConf, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConf, ok := existingConf.Data[constants.ClusterConfigurationKey] - if !ok { - return kubeadmv1beta3.ClusterConfiguration{}, nil, errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var existingClusterConfig kubeadmv1beta3.ClusterConfiguration - if err := yaml.Unmarshal([]byte(clusterConf), &existingClusterConfig); err != nil { - return kubeadmv1beta3.ClusterConfiguration{}, nil, fmt.Errorf("unmarshaling ClusterConfiguration: %w", err) - } - - return existingClusterConfig, existingConf, nil -} - // applyComponentsCM applies the k8s components ConfigMap to the cluster. func (k *KubeCmd) applyComponentsCM(ctx context.Context, components *corev1.ConfigMap) error { if err := k.retryAction(ctx, func(ctx context.Context) error { diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 74e9562c15..79eef03c34 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -676,6 +676,50 @@ func TestRetryAction(t *testing.T) { } } +func TestExtendClusterConfigCertSANs(t *testing.T) { + ctx := context.Background() + + testCases := map[string]struct { + clusterConfig string + }{ + "kubeadmv1beta3.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta3, + }, + "kubeadmv1beta4.ClusterConfiguration": { + clusterConfig: kubeadmClusterConfigurationV1Beta4, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + require := require.New(t) + assert := assert.New(t) + kubectl := &fakeConfigMapClient{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": tc.clusterConfig}}, + }, + } + cmd := &KubeCmd{ + kubectl: kubectl, + log: logger.NewTest(t), + retryInterval: time.Millisecond, + } + + err := cmd.ExtendClusterConfigCertSANs(ctx, []string{"example.com"}) + require.NoError(err) + + cm := kubectl.configMaps["kubeadm-config"] + require.NotNil(cm) + cc := cm.Data["ClusterConfiguration"] + require.NotNil(cc) + // Verify that SAN was added. + assert.Contains(cc, "example.com") + // Verify that config was written in v1beta4, regardless of the version read. + assert.Contains(cc, "kubeadm.k8s.io/v1beta4") + }) + } +} + type fakeUnstructuredClient struct { mock.Mock } @@ -835,3 +879,83 @@ func supportedValidK8sVersions() (res []versions.ValidK8sVersion) { } return } + +var kubeadmClusterConfigurationV1Beta3 = ` +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + kubelet-certificate-authority: /etc/kubernetes/pki/ca.crt + profiling: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + cloud-provider: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + profiling: "false" +` + +var kubeadmClusterConfigurationV1Beta4 = ` +apiVersion: kubeadm.k8s.io/v1beta4 +kind: ClusterConfiguration +apiServer: + certSANs: + - 127.0.0.1 + extraArgs: + - name: kubelet-certificate-authority + value: /etc/kubernetes/pki/ca.crt + - name: profiling + value: "false" + extraVolumes: + - hostPath: /var/log/kubernetes/audit/ + mountPath: /var/log/kubernetes/audit/ + name: audit-log + pathType: DirectoryOrCreate +certificatesDir: /etc/kubernetes/pki +clusterName: test-55bbf58d +controlPlaneEndpoint: 34.149.125.227:6443 +controllerManager: + extraArgs: + - name: cloud-provider + value: external +dns: + disabled: true +encryptionAlgorithm: RSA-2048 +etcd: + local: + dataDir: /var/lib/etcd +imageRepository: registry.k8s.io +kubernetesVersion: v1.31.1 +networking: + dnsDomain: cluster.local + serviceSubnet: 10.96.0.0/12 +proxy: + disabled: true +scheduler: + extraArgs: + - name: profiling + value: "false" +` diff --git a/internal/versions/versions.go b/internal/versions/versions.go index 858ff3ed48..b88f347256 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -100,12 +100,12 @@ func ResolveK8sPatchVersion(k8sVersion string) (string, error) { // supported patch version as PATCH. func k8sVersionFromMajorMinor(version string) string { switch version { - case semver.MajorMinor(string(V1_28)): - return string(V1_28) case semver.MajorMinor(string(V1_29)): return string(V1_29) case semver.MajorMinor(string(V1_30)): return string(V1_30) + case semver.MajorMinor(string(V1_31)): + return string(V1_31) default: return "" } @@ -181,14 +181,14 @@ const ( // currently supported versions. //nolint:revive - V1_28 ValidK8sVersion = "v1.28.15" // renovate:kubernetes-release - //nolint:revive V1_29 ValidK8sVersion = "v1.29.13" // renovate:kubernetes-release //nolint:revive V1_30 ValidK8sVersion = "v1.30.9" // renovate:kubernetes-release + //nolint:revive + V1_31 ValidK8sVersion = "v1.31.1" // renovate:kubernetes-release // Default k8s version deployed by Constellation. - Default ValidK8sVersion = V1_29 + Default ValidK8sVersion = V1_30 ) // Regenerate the hashes by running go generate. @@ -197,49 +197,49 @@ const ( // VersionConfigs holds download URLs for all required kubernetes components for every supported version. var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ - V1_28: { - ClusterVersion: "v1.28.15", // renovate:kubernetes-release + V1_31: { + ClusterVersion: "v1.31.1", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz", // renovate:cni-plugins-release - Hash: "sha256:682b49ff8933a997a52107161f1745f8312364b4c7f605ccdf7a77499130d89d", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release + Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.32.0/crictl-v1.32.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:f050b71d3a73a91a4e0990b90143ed04dcd100cc66f953736fcb6a2730e283c4", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://dl.k8s.io/v1.28.15/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:b07a27fd5bd2419c9c623de15c1dd339af84eb27e9276c81070071065db00036", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.28.15/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:0555b2c2fd30efcdb44b7fba5460c3dc3d3e39f2301e1eef7894a9f8976e1b4c", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.28.15/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:1f7651ad0b50ef4561aa82e77f3ad06599b5e6b0b2a5fb6c4f474d95a77e41c5", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI4LjE1QHNoYTI1Njo2ZGZhODRmNWQ2YmU3MTFhZTBkMTk3NTgyMDFkMzM3ZTgzNmFiN2RlNzMzMDZmZjE0NzI1Y2VhYTk3OGZlYThmIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI4LjE1QHNoYTI1NjpkYWRkMmEzNzg0NzgzMDE4YTdlZTg1ODhkMTFmNzg3ZmVlNGQ1NDI0ZjJjZGQ2Y2U4OWQzYmExODQ0YTZjMTc1In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI4LjE1QHNoYTI1Njo4MmY0YTQyMzE3NDUwODU4ZDNkNzBmZGU1YjNjMGYyMjE1M2VhMTU1ZmQwNTNmMDk4NjU5OTlhNDY2MWYyZGNhIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -249,20 +249,20 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.28.10@sha256:582571fc487b4ba52fbafe4385daec5d97f57cdd7f3e901211eef6411b2a90a6", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.4@sha256:47f861081efbc04bda32b6212ca2c74b5b2ce190e595a285e1b712ca0afec0c7", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.28.13@sha256:8b853f4f54a09c363806714189435933a8575ac6dca27e991976bd685603113e", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.28.13@sha256:525ca9c8a44bbdfa9acc0a417776bb822a1bbdaaf27d9776b8dcf5b3519c346a", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v28.10.0@sha256:f3b6fa7faea27b4a303c91b3bc7ee192b050e21e27579e9f3da90ae4ba38e626", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container + CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.28.7@sha256:77906954da9171425c8c8d3286091818143b6dcf9039abd49b8f33f1502978a1", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container }, V1_29: { ClusterVersion: "v1.29.13", // renovate:kubernetes-release diff --git a/terraform-provider-constellation/docs/resources/cluster.md b/terraform-provider-constellation/docs/resources/cluster.md index eb4ed95e50..b2ba672860 100644 --- a/terraform-provider-constellation/docs/resources/cluster.md +++ b/terraform-provider-constellation/docs/resources/cluster.md @@ -69,7 +69,7 @@ resource "constellation_cluster" "azure_example" { See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview/clouds) that Constellation supports. - `image` (Attributes) Constellation OS Image to use on the nodes. (see [below for nested schema](#nestedatt--image)) - `init_secret` (String) Secret used for initialization of the cluster. -- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.28.15 v1.29.13 v1.30.9]. +- `kubernetes_version` (String) The Kubernetes version to use for the cluster. The supported versions are [v1.29.13 v1.30.9 v1.31.1]. - `master_secret` (String) Hex-encoded 32-byte master secret for the cluster. - `master_secret_salt` (String) Hex-encoded 32-byte master secret salt for the cluster. - `measurement_salt` (String) Hex-encoded 32-byte measurement salt for the cluster. From 73604694372d7c4f3378c63117c9b055fc46aeee Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:22:17 +0200 Subject: [PATCH 2/6] e2e: set default k8s version for daily to 1.30 --- .github/workflows/e2e-test-daily.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e-test-daily.yml b/.github/workflows/e2e-test-daily.yml index e9ce79e11d..a618b44a6b 100644 --- a/.github/workflows/e2e-test-daily.yml +++ b/.github/workflows/e2e-test-daily.yml @@ -45,7 +45,7 @@ jobs: fail-fast: false max-parallel: 5 matrix: - kubernetesVersion: ["1.29"] # should be default + kubernetesVersion: ["1.30"] # This should correspond to the current default k8s minor. attestationVariant: ["gcp-sev-es", "gcp-sev-snp", "azure-sev-snp", "azure-tdx", "aws-sev-snp"] refStream: ["ref/main/stream/debug/?", "ref/release/stream/stable/?"] test: ["sonobuoy quick"] From 331944e925bf85940341f399bb032a3c84dfffa1 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:22:42 +0200 Subject: [PATCH 3/6] e2e: remove defaults for required arguments --- .github/workflows/e2e-test-internal-lb.yml | 1 - .github/workflows/e2e-test-marketplace-image.yml | 1 - .github/workflows/e2e-test-terraform-provider.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/.github/workflows/e2e-test-internal-lb.yml b/.github/workflows/e2e-test-internal-lb.yml index 0371605894..c31b756786 100644 --- a/.github/workflows/e2e-test-internal-lb.yml +++ b/.github/workflows/e2e-test-internal-lb.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-marketplace-image.yml b/.github/workflows/e2e-test-marketplace-image.yml index d0f9e1f533..84cd8d9941 100644 --- a/.github/workflows/e2e-test-marketplace-image.yml +++ b/.github/workflows/e2e-test-marketplace-image.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true cliVersion: description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref." diff --git a/.github/workflows/e2e-test-terraform-provider.yml b/.github/workflows/e2e-test-terraform-provider.yml index 5054bab186..9584a69196 100644 --- a/.github/workflows/e2e-test-terraform-provider.yml +++ b/.github/workflows/e2e-test-terraform-provider.yml @@ -41,7 +41,6 @@ on: required: true kubernetesVersion: description: "Kubernetes version to create the cluster from." - default: "1.29" required: true releaseVersion: description: "Version of a released provider to download. Leave empty to build the provider from the checked out ref." From a73483f11a583fbaec4d54d9b6cedec1465be6d4 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 9 Oct 2024 09:14:13 +0200 Subject: [PATCH 4/6] versions: move 1.31 to the end of the list --- internal/versions/versions.go | 116 +++++++++++++++++----------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/internal/versions/versions.go b/internal/versions/versions.go index b88f347256..4cbcdf9034 100644 --- a/internal/versions/versions.go +++ b/internal/versions/versions.go @@ -197,49 +197,49 @@ const ( // VersionConfigs holds download URLs for all required kubernetes components for every supported version. var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ - V1_31: { - ClusterVersion: "v1.31.1", // renovate:kubernetes-release + V1_29: { + ClusterVersion: "v1.29.13", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release - Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz", // renovate:cni-plugins-release + Hash: "sha256:682b49ff8933a997a52107161f1745f8312364b4c7f605ccdf7a77499130d89d", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.32.0/crictl-v1.32.0-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:f050b71d3a73a91a4e0990b90143ed04dcd100cc66f953736fcb6a2730e283c4", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", + Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:ed89d8cdf60b14aae2aa665f8c1e3fc3e868ccc511c8ef5916f55c8cbd5ec772", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", + Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:584d4ded69ea6e660d7b205c9c924f37cdfca24ad858cf05aeca626048e25f46", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", + Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:9f4cba9b3e12a3fd7fa99dee651d7293281333469852a8e755a1210d5b128b8d", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjEzQHNoYTI1NjplNWM0Mjg2MTA0NWQwNjE1NzY5ZmFkOGE0ZTMyZTQ3NmZjNWU1OTAyMDE1N2I2MGNlZDFiYjdhNjlkNGE1Y2U5In1d", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjEzQHNoYTI1NjpmYzI4MzgzOTk3NTI3NDBiZGQzNmM3ZTkyODdkNDQwNmZlZmY2YmVmMmJhZmYzOTMxNzRiMzRjY2Q0NDdiNzgwIn1d", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjEzQHNoYTI1NjphNGYxNjQ5YTUyNDljMDc4NDk2M2Q4NTY0NGIxZTYxNDU0OGYwMzJkYTliNGZiMDBhNzYwYmFjMDI4MThjZTRmIn1d", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -249,23 +249,23 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.4@sha256:47f861081efbc04bda32b6212ca2c74b5b2ce190e595a285e1b712ca0afec0c7", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.7@sha256:5dfb0bf6bbfa99e0f572bb8b65fbb36576c4f256499e63371b550353702c0483", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.11@sha256:2ecdca660c03b17110a4ee732230424ce0377c5b1756a4408666e40938ee976a", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.11@sha256:17888b0ebaec6735214b85d20bdcc8062f051bc27e835454e9ef89734d34aa4b", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container + CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.5@sha256:76865740be7c965b35ee1524931bb4abfe4c27b5bfad280e84068cd6653ee7bb", // renovate:container }, - V1_29: { - ClusterVersion: "v1.29.13", // renovate:kubernetes-release + V1_30: { + ClusterVersion: "v1.30.9", // renovate:kubernetes-release KubernetesComponents: components.Components{ { Url: "https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz", // renovate:cni-plugins-release @@ -280,33 +280,33 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ Extract: true, }, { - Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:ed89d8cdf60b14aae2aa665f8c1e3fc3e868ccc511c8ef5916f55c8cbd5ec772", + Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:59c61c8018686bb58be96481f8aa3abe42bba9f791c6dd3c9d4a2ed697187e5b", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:584d4ded69ea6e660d7b205c9c924f37cdfca24ad858cf05aeca626048e25f46", + Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:f763a8f5616cf1da80d88555b3654ab6aacd62dc62e6ba7dd2d540c34eea24c0", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.29.13/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:9f4cba9b3e12a3fd7fa99dee651d7293281333469852a8e755a1210d5b128b8d", + Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:d77041f285d9237c4aa451370c3ec6e5c042007dbb55c894f0a179b1d149bf32", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjI5LjEzQHNoYTI1NjplNWM0Mjg2MTA0NWQwNjE1NzY5ZmFkOGE0ZTMyZTQ3NmZjNWU1OTAyMDE1N2I2MGNlZDFiYjdhNjlkNGE1Y2U5In1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjlAc2hhMjU2OjU0MGRlOGY4MTBhYzk2M2I4ZWQ5M2Y3MzkzYTg3NDZkNjhlN2U4YTJjNzllYTU4ZmY0MDlhYzViOWNhNmE5ZmMifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjI5LjEzQHNoYTI1NjpmYzI4MzgzOTk3NTI3NDBiZGQzNmM3ZTkyODdkNDQwNmZlZmY2YmVmMmJhZmYzOTMxNzRiMzRjY2Q0NDdiNzgwIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjlAc2hhMjU2OjYzNTA2OTNjMDQ5NTZiMTNkYjI1MTllMDFjYTEyYTBiYmU1ODQ2NmU5ZjEyZWY4NjE3ZjE0MjlkYTYwODFmNDMifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjI5LjEzQHNoYTI1NjphNGYxNjQ5YTUyNDljMDc4NDk2M2Q4NTY0NGIxZTYxNDU0OGYwMzJkYTliNGZiMDBhNzYwYmFjMDI4MThjZTRmIn1d", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjlAc2hhMjU2OjE1M2VmZDZkYzg5ZTYxYTM4ZWYyNzNjZjRjNGNlYmQyYmZlZTY4MDgyYzJlZTNkNGZhYjVkYTk0ZTRhZTEzZDMifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -316,64 +316,64 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.29.7@sha256:5dfb0bf6bbfa99e0f572bb8b65fbb36576c4f256499e63371b550353702c0483", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.6@sha256:cfdfa9e436f27fccfd3f0961e9607088482b17e43e2e1990e02e925a833f0ef3", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.29.11@sha256:2ecdca660c03b17110a4ee732230424ce0377c5b1756a4408666e40938ee976a", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.7@sha256:03b2876f481507781a27b56a6e66c1928b7b93774e787e52a5239aefa41191e4", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.29.11@sha256:17888b0ebaec6735214b85d20bdcc8062f051bc27e835454e9ef89734d34aa4b", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.7@sha256:f18feb78e36eef88f0e23d98d798476d2bf6837de11892fe118ab043afdcd497", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v29.5.1@sha256:ebbc6f5755725b6c2c81ca1d1580e2feba83572c41608b739c50f85b2e5de936", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.4@sha256:0c3695a18d3825492196facb092e5fe56e466fa8517cde5a206fe21630c1da13", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.29.5@sha256:76865740be7c965b35ee1524931bb4abfe4c27b5bfad280e84068cd6653ee7bb", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.3@sha256:08fd86ee093760849ac4fd579eb90185b669fc20aa56c156aa34ea7b73dd5e34", // renovate:container }, - V1_30: { - ClusterVersion: "v1.30.9", // renovate:kubernetes-release + V1_31: { + ClusterVersion: "v1.31.1", // renovate:kubernetes-release KubernetesComponents: components.Components{ { - Url: "https://github.com/containernetworking/plugins/releases/download/v1.6.0/cni-plugins-linux-amd64-v1.6.0.tgz", // renovate:cni-plugins-release - Hash: "sha256:682b49ff8933a997a52107161f1745f8312364b4c7f605ccdf7a77499130d89d", + Url: "https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz", // renovate:cni-plugins-release + Hash: "sha256:77baa2f669980a82255ffa2f2717de823992480271ee778aa51a9c60ae89ff9b", InstallPath: constants.CniPluginsDir, Extract: true, }, { - Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.32.0/crictl-v1.32.0-linux-amd64.tar.gz", // renovate:crictl-release - Hash: "sha256:f050b71d3a73a91a4e0990b90143ed04dcd100cc66f953736fcb6a2730e283c4", + Url: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz", // renovate:crictl-release + Hash: "sha256:0a03ba6b1e4c253d63627f8d210b2ea07675a8712587e697657b236d06d7d231", InstallPath: constants.BinDir, Extract: true, }, { - Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubelet", // renovate:kubernetes-release - Hash: "sha256:59c61c8018686bb58be96481f8aa3abe42bba9f791c6dd3c9d4a2ed697187e5b", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubelet", // renovate:kubernetes-release + Hash: "sha256:50619fff95bdd7e690c049cc083f495ae0e7c66d0cdf6a8bcad298af5fe28438", InstallPath: constants.KubeletPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubeadm", // renovate:kubernetes-release - Hash: "sha256:f763a8f5616cf1da80d88555b3654ab6aacd62dc62e6ba7dd2d540c34eea24c0", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubeadm", // renovate:kubernetes-release + Hash: "sha256:b3f92d19d482359116dd9ee9c0a10cb86e32a2a2aef79b853d5f07d6a093b0df", InstallPath: constants.KubeadmPath, Extract: false, }, { - Url: "https://dl.k8s.io/v1.30.9/bin/linux/amd64/kubectl", // renovate:kubernetes-release - Hash: "sha256:d77041f285d9237c4aa451370c3ec6e5c042007dbb55c894f0a179b1d149bf32", + Url: "https://dl.k8s.io/v1.31.1/bin/linux/amd64/kubectl", // renovate:kubernetes-release + Hash: "sha256:57b514a7facce4ee62c93b8dc21fda8cf62ef3fed22e44ffc9d167eab843b2ae", InstallPath: constants.KubectlPath, Extract: false, }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMwLjlAc2hhMjU2OjU0MGRlOGY4MTBhYzk2M2I4ZWQ5M2Y3MzkzYTg3NDZkNjhlN2U4YTJjNzllYTU4ZmY0MDlhYzViOWNhNmE5ZmMifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtYXBpc2VydmVyOnYxLjMxLjFAc2hhMjU2OjI0MDljMjNkYmI1YTJiN2E4MWFkYmIxODRkM2VhYzQzYWM2NTNlOWI5N2E3YzBlZTEyMWI4OWJiM2VmNjFmZGIifV0=", InstallPath: patchFilePath("kube-apiserver"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMwLjlAc2hhMjU2OjYzNTA2OTNjMDQ5NTZiMTNkYjI1MTllMDFjYTEyYTBiYmU1ODQ2NmU5ZjEyZWY4NjE3ZjE0MjlkYTYwODFmNDMifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtY29udHJvbGxlci1tYW5hZ2VyOnYxLjMxLjFAc2hhMjU2OjlmOWRhNWIyN2UwM2Y4OTU5OWNjNDBiYTg5MTUwYWViZjNiNGNmZjAwMWU2ZGI2ZDk5ODY3NGIzNDE4MWUxYTEifV0=", InstallPath: patchFilePath("kube-controller-manager"), }, { - Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMwLjlAc2hhMjU2OjE1M2VmZDZkYzg5ZTYxYTM4ZWYyNzNjZjRjNGNlYmQyYmZlZTY4MDgyYzJlZTNkNGZhYjVkYTk0ZTRhZTEzZDMifV0=", + Url: "data:application/json;base64,W3sib3AiOiJyZXBsYWNlIiwicGF0aCI6Ii9zcGVjL2NvbnRhaW5lcnMvMC9pbWFnZSIsInZhbHVlIjoicmVnaXN0cnkuazhzLmlvL2t1YmUtc2NoZWR1bGVyOnYxLjMxLjFAc2hhMjU2Ojk2OWE3ZTk2MzQwZjNhOTI3YjNkNjUyNTgyZWRlYzJkNmQ4MmEwODM4NzFkODFlZjUwNjRiN2VkYWFiNDMwZDAifV0=", InstallPath: patchFilePath("kube-scheduler"), }, { @@ -383,20 +383,20 @@ var VersionConfigs = map[ValidK8sVersion]KubernetesVersion{ }, // CloudControllerManagerImageAWS is the CCM image used on AWS. // Check for newer versions at https://github.com/kubernetes/cloud-provider-aws/releases. - CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.30.6@sha256:cfdfa9e436f27fccfd3f0961e9607088482b17e43e2e1990e02e925a833f0ef3", // renovate:container + CloudControllerManagerImageAWS: "registry.k8s.io/provider-aws/cloud-controller-manager:v1.31.4@sha256:47f861081efbc04bda32b6212ca2c74b5b2ce190e595a285e1b712ca0afec0c7", // renovate:container // CloudControllerManagerImageAzure is the CCM image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.30.7@sha256:03b2876f481507781a27b56a6e66c1928b7b93774e787e52a5239aefa41191e4", // renovate:container + CloudControllerManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:v1.31.1@sha256:b5aa55a7e9d38137f7fcd0adc9335b06e7c96061764addd7e6bb9f86403f0110", // renovate:container // CloudNodeManagerImageAzure is the cloud-node-manager image used on Azure. // Check for newer versions at https://github.com/kubernetes-sigs/cloud-provider-azure/blob/master/README.md. - CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.30.7@sha256:f18feb78e36eef88f0e23d98d798476d2bf6837de11892fe118ab043afdcd497", // renovate:container + CloudNodeManagerImageAzure: "mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager:v1.31.1@sha256:e9b522399e4ec6bc4ce90c173e59db135d742de7b16f0f5454b4d88ba78a98c7", // renovate:container // CloudControllerManagerImageGCP is the CCM image used on GCP. - CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.4@sha256:0c3695a18d3825492196facb092e5fe56e466fa8517cde5a206fe21630c1da13", // renovate:container + CloudControllerManagerImageGCP: "ghcr.io/edgelesssys/cloud-provider-gcp:v30.1.0@sha256:64d2d5d4d2b5fb426c307c64ada9a61b64e797b56d9768363f145f2bd957998f", // renovate:container // CloudControllerManagerImageOpenStack is the CCM image used on OpenStack. - CloudControllerManagerImageOpenStack: "docker.io/k8scloudprovider/openstack-cloud-controller-manager:v1.26.4@sha256:05e846fb13481b6dbe4a1e50491feb219e8f5101af6cf662a086115735624db0", // renovate:container + CloudControllerManagerImageOpenStack: "registry.k8s.io/provider-os/openstack-cloud-controller-manager:v1.31.1@sha256:72cc0d22b83c613df809d8134e50404171513d92287e63e2313d9ad7e1ed630e", // renovate:container // External service image. Depends on k8s version. // Check for new versions at https://github.com/kubernetes/autoscaler/releases. - ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.30.3@sha256:08fd86ee093760849ac4fd579eb90185b669fc20aa56c156aa34ea7b73dd5e34", // renovate:container + ClusterAutoscalerImage: "registry.k8s.io/autoscaling/cluster-autoscaler:v1.31.0@sha256:6d4c51c35f344d230341d71bb6d35f2c2f0c0a6f205a7887ae44e6d852fb5b5f", // renovate:container }, } From 3e22aa8d43963c3b535f99e1ea82f43a7dc5facb Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Thu, 10 Oct 2024 20:04:08 +0200 Subject: [PATCH 5/6] kubernetes: set feature gate ControlPlaneKubeletLocalMode Co-Authored-By: Leonard Cohnen --- .../internal/kubernetes/k8sapi/BUILD.bazel | 1 + .../kubernetes/k8sapi/kubeadm_config.go | 8 +- internal/constellation/kubecmd/kubecmd.go | 132 +++++++++++------- .../constellation/kubecmd/kubecmd_test.go | 3 + 4 files changed, 89 insertions(+), 55 deletions(-) diff --git a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel index ef87085bd8..968a2b0114 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel +++ b/bootstrapper/internal/kubernetes/k8sapi/BUILD.bazel @@ -28,6 +28,7 @@ go_library( "@io_k8s_kubelet//config/v1beta1", "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_kubernetes//cmd/kubeadm/app/constants", + "@org_golang_x_mod//semver", ], ) diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go index 430839a469..1ce387aeb4 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubeadm_config.go @@ -12,6 +12,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/certificate" "github.com/edgelesssys/constellation/v2/internal/constants" "github.com/edgelesssys/constellation/v2/internal/kubernetes" + "golang.org/x/mod/semver" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kubeletconf "k8s.io/kubelet/config/v1beta1" @@ -38,7 +39,7 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl cloudProvider = "external" } - return KubeadmInitYAML{ + initConfig := KubeadmInitYAML{ InitConfiguration: kubeadm.InitConfiguration{ TypeMeta: metav1.TypeMeta{ APIVersion: kubeadm.SchemeGroupVersion.String(), @@ -157,6 +158,11 @@ func (c *KubdeadmConfiguration) InitConfiguration(externalCloudProvider bool, cl TLSPrivateKeyFile: certificate.KeyFilename, }, } + + if semver.Compare(clusterVersion, "v1.31.0") >= 0 { + initConfig.ClusterConfiguration.FeatureGates = map[string]bool{"ControlPlaneKubeletLocalMode": true} + } + return initConfig } // JoinConfiguration returns a new kubeadm join configuration. diff --git a/internal/constellation/kubecmd/kubecmd.go b/internal/constellation/kubecmd/kubecmd.go index a193f2528e..dd3c4f9da7 100644 --- a/internal/constellation/kubecmd/kubecmd.go +++ b/internal/constellation/kubecmd/kubecmd.go @@ -131,6 +131,18 @@ func (k *KubeCmd) UpgradeKubernetesVersion(ctx context.Context, kubernetesVersio ) } + // TODO(burgerdev): remove after releasing v2.19 + // Workaround for https://github.com/kubernetes/kubernetes/issues/127316: force kubelet to + // connect to the local API server. + if err := k.patchKubeadmConfig(ctx, func(cc *kubeadm.ClusterConfiguration) { + if cc.FeatureGates == nil { + cc.FeatureGates = map[string]bool{} + } + cc.FeatureGates["ControlPlaneKubeletLocalMode"] = true + }); err != nil { + return fmt.Errorf("setting FeatureGate ControlPlaneKubeletLocalMode: %w", err) + } + versionConfig, ok := versions.VersionConfigs[kubernetesVersion] if !ok { return fmt.Errorf("skipping Kubernetes upgrade: %w", compatibility.NewInvalidUpgradeError( @@ -236,65 +248,32 @@ func (k *KubeCmd) ApplyJoinConfig(ctx context.Context, newAttestConfig config.At // ExtendClusterConfigCertSANs extends the ClusterConfig stored under "kube-system/kubeadm-config" with the given SANs. // Empty strings are ignored, existing SANs are preserved. func (k *KubeCmd) ExtendClusterConfigCertSANs(ctx context.Context, alternativeNames []string) error { - var kubeadmConfig *corev1.ConfigMap - if err := k.retryAction(ctx, func(ctx context.Context) error { - var err error - kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) - return err - }); err != nil { - return fmt.Errorf("retrieving current kubeadm-config: %w", err) - } - - clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] - if !ok { - return errors.New("ClusterConfiguration missing from kubeadm-config") - } - - var clusterConfiguration kubeadm.ClusterConfiguration - if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { - return fmt.Errorf("decoding cluster configuration data: %w", err) - } - - existingSANs := make(map[string]struct{}) - for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { - existingSANs[existingSAN] = struct{}{} - } - - var missingSANs []string - for _, san := range alternativeNames { - if san == "" { - continue // skip empty SANs + if err := k.patchKubeadmConfig(ctx, func(clusterConfiguration *kubeadm.ClusterConfiguration) { + existingSANs := make(map[string]struct{}) + for _, existingSAN := range clusterConfiguration.APIServer.CertSANs { + existingSANs[existingSAN] = struct{}{} } - if _, ok := existingSANs[san]; !ok { - missingSANs = append(missingSANs, san) - existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice - } - } - if len(missingSANs) == 0 { - k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") - return nil - } - k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - - clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) - sort.Strings(clusterConfiguration.APIServer.CertSANs) + var missingSANs []string + for _, san := range alternativeNames { + if san == "" { + continue // skip empty SANs + } + if _, ok := existingSANs[san]; !ok { + missingSANs = append(missingSANs, san) + existingSANs[san] = struct{}{} // make sure we don't add the same SAN twice + } + } - opt := k8sjson.SerializerOptions{Yaml: true} - serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) - encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) - newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) - if err != nil { - return fmt.Errorf("marshaling ClusterConfiguration: %w", err) - } + if len(missingSANs) == 0 { + k.log.Debug("No new SANs to add to the cluster's apiserver SAN field") + } + k.log.Debug("Extending the cluster's apiserver SAN field", "certSANs", strings.Join(missingSANs, ", ")) - kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) - k.log.Debug("Triggering kubeadm config update now") - if err = k.retryAction(ctx, func(ctx context.Context) error { - _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) - return err + clusterConfiguration.APIServer.CertSANs = append(clusterConfiguration.APIServer.CertSANs, missingSANs...) + sort.Strings(clusterConfiguration.APIServer.CertSANs) }); err != nil { - return fmt.Errorf("setting new kubeadm config: %w", err) + return fmt.Errorf("extending ClusterConfig.CertSANs: %w", err) } k.log.Debug("Successfully extended the cluster's apiserver SAN field") @@ -462,6 +441,51 @@ func (k *KubeCmd) retryAction(ctx context.Context, action func(ctx context.Conte return retrier.Do(ctx) } +// patchKubeadmConfig fetches and unpacks the kube-system/kubeadm-config ClusterConfiguration entry, +// runs doPatch on it and uploads the result. +func (k *KubeCmd) patchKubeadmConfig(ctx context.Context, doPatch func(*kubeadm.ClusterConfiguration)) error { + var kubeadmConfig *corev1.ConfigMap + if err := k.retryAction(ctx, func(ctx context.Context) error { + var err error + kubeadmConfig, err = k.kubectl.GetConfigMap(ctx, constants.ConstellationNamespace, constants.KubeadmConfigMap) + return err + }); err != nil { + return fmt.Errorf("retrieving current kubeadm-config: %w", err) + } + + clusterConfigData, ok := kubeadmConfig.Data[constants.ClusterConfigurationKey] + if !ok { + return errors.New("ClusterConfiguration missing from kubeadm-config") + } + + var clusterConfiguration kubeadm.ClusterConfiguration + if err := runtime.DecodeInto(kubeadmscheme.Codecs.UniversalDecoder(), []byte(clusterConfigData), &clusterConfiguration); err != nil { + return fmt.Errorf("decoding cluster configuration data: %w", err) + } + + doPatch(&clusterConfiguration) + + opt := k8sjson.SerializerOptions{Yaml: true} + serializer := k8sjson.NewSerializerWithOptions(k8sjson.DefaultMetaFactory, kubeadmscheme.Scheme, kubeadmscheme.Scheme, opt) + encoder := kubeadmscheme.Codecs.EncoderForVersion(serializer, kubeadmv1beta4.SchemeGroupVersion) + newConfigYAML, err := runtime.Encode(encoder, &clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debug("Triggering kubeadm config update now") + if err = k.retryAction(ctx, func(ctx context.Context) error { + _, err := k.kubectl.UpdateConfigMap(ctx, kubeadmConfig) + return err + }); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + k.log.Debug("Successfully patched the cluster's kubeadm-config") + return nil +} + func checkForApplyError(expected, actual updatev1alpha1.NodeVersion) error { var err error switch { diff --git a/internal/constellation/kubecmd/kubecmd_test.go b/internal/constellation/kubecmd/kubecmd_test.go index 79eef03c34..c88dab9bda 100644 --- a/internal/constellation/kubecmd/kubecmd_test.go +++ b/internal/constellation/kubecmd/kubecmd_test.go @@ -281,6 +281,9 @@ func TestUpgradeKubernetesVersion(t *testing.T) { } kubectl := &stubKubectl{ unstructuredInterface: unstructuredClient, + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: {Data: map[string]string{"ClusterConfiguration": kubeadmClusterConfigurationV1Beta4}}, + }, } if tc.customClientFn != nil { kubectl.unstructuredInterface = tc.customClientFn(nodeVersion) From d5860bf861c600c0fc5ac272b8b958361164d022 Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Mon, 23 Dec 2024 11:00:52 +0100 Subject: [PATCH 6/6] operator: update control plane nodes first --- .../controllers/nodeversion_controller.go | 18 +++++++++ .../nodeversion_controller_test.go | 40 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller.go b/operators/constellation-node-operator/controllers/nodeversion_controller.go index ff706c702f..276448f14d 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "reflect" + "slices" "strings" "time" @@ -388,6 +389,10 @@ func (r *NodeVersionReconciler) tryStartClusterVersionUpgrade(ctx context.Contex func (r *NodeVersionReconciler) pairDonorsAndHeirs(ctx context.Context, controller metav1.Object, outdatedNodes []corev1.Node, mintNodes []mintNode) []replacementPair { logr := log.FromContext(ctx) var pairs []replacementPair + + // Prioritize control-plane nodes, which need to be upgraded first starting with k8s v1.31.0. + sortByControlPlanes(outdatedNodes) + for _, mintNode := range mintNodes { var foundReplacement bool // find outdated node in the same group @@ -943,3 +948,16 @@ type newNodeConfig struct { scalingGroupByID map[string]updatev1alpha1.ScalingGroup newNodesBudget int } + +func sortByControlPlanes(nodes []corev1.Node) { + slices.SortStableFunc(nodes, func(a, b corev1.Node) int { + _, aControlPlane := a.Labels["node-role.kubernetes.io/control-plane"] + _, bControlPlane := b.Labels["node-role.kubernetes.io/control-plane"] + if aControlPlane == bControlPlane { + return 0 + } else if aControlPlane { + return -1 + } + return 1 + }) +} diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go index c9ae880421..5146c4cdcf 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller_test.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller_test.go @@ -9,6 +9,7 @@ package controllers import ( "context" "errors" + "fmt" "sync" "testing" @@ -891,3 +892,42 @@ func (*unimplementedNodeReplacer) CreateNode(_ context.Context, _ string) (nodeN func (*unimplementedNodeReplacer) DeleteNode(_ context.Context, _ string) error { panic("unimplemented") } + +func TestSortByControlPlane(t *testing.T) { + w1 := newNode("w1", false) + w2 := newNode("w2", false) + cp1 := newNode("cp1", true) + cp2 := newNode("cp2", true) + + for i, tc := range []struct { + input []corev1.Node + expected []string + }{ + {input: []corev1.Node{w1, w2, cp1, cp2}, expected: []string{"cp1", "cp2", "w1", "w2"}}, + {input: []corev1.Node{w2, cp1, cp2, w1}, expected: []string{"cp1", "cp2", "w2", "w1"}}, + {input: []corev1.Node{cp2, w1, cp1, w2}, expected: []string{"cp2", "cp1", "w1", "w2"}}, + {input: []corev1.Node{cp1, cp2, w1, w2}, expected: []string{"cp1", "cp2", "w1", "w2"}}, + } { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + sortByControlPlanes(tc.input) + var actual []string + for _, node := range tc.input { + actual = append(actual, node.Name) + } + assert.Equal(t, tc.expected, actual) + }) + } +} + +func newNode(name string, controlPlane bool) corev1.Node { + node := corev1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: make(map[string]string), + }, + } + if controlPlane { + node.Labels["node-role.kubernetes.io/control-plane"] = "" + } + return node +}