From 05a03ce92cf461ef1fc30d7a3fba0cf1dbf2a172 Mon Sep 17 00:00:00 2001 From: Ekaterina Kazakova Date: Thu, 14 Nov 2024 16:56:28 +0400 Subject: [PATCH] [e2e] Support to upgrade managed clusters --- test/e2e/kubeclient/kubeclient.go | 4 + test/e2e/managedcluster/upgrade.go | 89 +++++++++++++++++++++ test/e2e/scenarios/provider_aws_test.go | 18 +++++ test/e2e/scenarios/provider_azure_test.go | 18 +++++ test/e2e/scenarios/provider_vsphere_test.go | 9 +++ 5 files changed, 138 insertions(+) create mode 100644 test/e2e/managedcluster/upgrade.go diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index 12cd3808e..281060be6 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -21,6 +21,7 @@ import ( "os" "path/filepath" + hcv2 "github.com/fluxcd/helm-controller/api/v2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -141,6 +142,9 @@ func newKubeClient(configBytes []byte, namespace string) *KubeClient { err = hmcmirantiscomv1alpha1.AddToScheme(scheme) Expect(err).NotTo(HaveOccurred(), "failed to add HMC API to scheme") + err = hcv2.AddToScheme(scheme) + Expect(err).NotTo(HaveOccurred(), "failed to add Flux helm controller API to scheme") + crClient, err := crclient.New(config, crclient.Options{Scheme: scheme}) Expect(err).NotTo(HaveOccurred(), "failed to create controller runtime client") diff --git a/test/e2e/managedcluster/upgrade.go b/test/e2e/managedcluster/upgrade.go new file mode 100644 index 000000000..ae04ded76 --- /dev/null +++ b/test/e2e/managedcluster/upgrade.go @@ -0,0 +1,89 @@ +// Copyright 2024 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package managedcluster + +import ( + "context" + "fmt" + "time" + + hcv2 "github.com/fluxcd/helm-controller/api/v2" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + apimeta "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + crclient "sigs.k8s.io/controller-runtime/pkg/client" + + hmc "github.com/Mirantis/hmc/api/v1alpha1" +) + +func Upgrade(ctx context.Context, cl crclient.Client, clusterNamespace, clusterName, newTemplate string) { + cluster := &hmc.ManagedCluster{} + err := cl.Get(ctx, types.NamespacedName{ + Namespace: clusterNamespace, + Name: clusterName, + }, cluster) + Expect(err).NotTo(HaveOccurred()) + + patch := crclient.MergeFrom(cluster.DeepCopy()) + cluster.Spec.Template = newTemplate + err = cl.Patch(ctx, cluster, patch) + Expect(err).NotTo(HaveOccurred()) + + template := &hmc.ClusterTemplate{} + err = cl.Get(ctx, types.NamespacedName{ + Namespace: clusterNamespace, + Name: newTemplate, + }, template) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func() bool { + errorMessage, upgraded := validateClusterUpgraded(ctx, cl, clusterNamespace, clusterName, template.Status.ChartRef.Name) + if !upgraded { + _, _ = fmt.Fprintf(GinkgoWriter, errorMessage, "\n") + return false + } + return true + }, 20*time.Minute, 20*time.Second).Should(BeTrue()) +} + +func validateClusterUpgraded(ctx context.Context, cl crclient.Client, clusterNamespace, clusterName, chartName string) (string, bool) { + hr := &hcv2.HelmRelease{} + err := cl.Get(ctx, types.NamespacedName{ + Namespace: clusterNamespace, + Name: clusterName, + }, hr) + if err != nil { + return fmt.Sprintf("failed to get %s/%s HelmRelease %v", clusterNamespace, clusterName, err), false + } + if hr.Spec.ChartRef.Name != chartName { + return fmt.Sprintf("waiting for chartName to be updated in %s/%s HelmRelease", clusterNamespace, clusterName), false + } + readyCondition := apimeta.FindStatusCondition(hr.GetConditions(), hmc.ReadyCondition) + if readyCondition == nil { + return fmt.Sprintf("waiting for %s/%s HelmRelease to have Ready condition", clusterNamespace, clusterName), false + } + if readyCondition.ObservedGeneration != hr.Generation { + return "waiting for status.observedGeneration to be updated", false + } + if readyCondition.Status != metav1.ConditionTrue { + return "waiting for Ready condition to have status: true", false + } + if readyCondition.Reason != hcv2.UpgradeSucceededReason { + return "waiting for Ready condition to have `UpgradeSucceeded` reason", false + } + return "", true +} diff --git a/test/e2e/scenarios/provider_aws_test.go b/test/e2e/scenarios/provider_aws_test.go index c454ffea1..155c9c263 100644 --- a/test/e2e/scenarios/provider_aws_test.go +++ b/test/e2e/scenarios/provider_aws_test.go @@ -176,6 +176,24 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return deploymentValidator.Validate(context.Background(), standaloneClient) }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + if testingConfig.Standalone.Upgrade { + managedcluster.Upgrade(ctx, kc.CrClient, managedcluster.Namespace, clusterName, testingConfig.Standalone.UpgradeTemplate) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + // Validate hosted deployment + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + } + if testingConfig.Hosted.Upgrade { + managedcluster.Upgrade(ctx, standaloneClient.CrClient, managedcluster.Namespace, hdName, testingConfig.Hosted.UpgradeTemplate) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + } + // Delete the hosted ManagedCluster and verify it is removed. templateBy(templates.TemplateAWSHostedCP, "deleting the ManagedCluster") err = hostedDeleteFunc() diff --git a/test/e2e/scenarios/provider_azure_test.go b/test/e2e/scenarios/provider_azure_test.go index b7b0a5a46..281ab27d4 100644 --- a/test/e2e/scenarios/provider_azure_test.go +++ b/test/e2e/scenarios/provider_azure_test.go @@ -169,6 +169,24 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return deploymentValidator.Validate(context.Background(), standaloneClient) }).WithTimeout(90 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + if testingConfig.Standalone.Upgrade { + managedcluster.Upgrade(ctx, kc.CrClient, managedcluster.Namespace, sdName, testingConfig.Standalone.UpgradeTemplate) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + // Validate hosted deployment + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + } + if testingConfig.Hosted.Upgrade { + managedcluster.Upgrade(ctx, standaloneClient.CrClient, managedcluster.Namespace, hdName, testingConfig.Hosted.UpgradeTemplate) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + } + By("verify the deployment deletes successfully") err = hostedDeleteFunc() Expect(err).NotTo(HaveOccurred()) diff --git a/test/e2e/scenarios/provider_vsphere_test.go b/test/e2e/scenarios/provider_vsphere_test.go index acbb6ab2e..3a7cd8f61 100644 --- a/test/e2e/scenarios/provider_vsphere_test.go +++ b/test/e2e/scenarios/provider_vsphere_test.go @@ -33,6 +33,8 @@ import ( ) var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere"), Ordered, func() { + ctx := context.Background() + var ( kc *kubeclient.KubeClient deleteFunc func() error @@ -113,5 +115,12 @@ var _ = Context("vSphere Templates", Label("provider:onprem", "provider:vsphere" Eventually(func() error { return deploymentValidator.Validate(context.Background(), kc) }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + if testingConfig.Standalone.Upgrade { + managedcluster.Upgrade(ctx, kc.CrClient, managedcluster.Namespace, clusterName, testingConfig.Standalone.UpgradeTemplate) + Eventually(func() error { + return deploymentValidator.Validate(context.Background(), kc) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + } }) })