[e2e] Support managed cluster upgrade testing

Makefile

@@ -71,6 +71,7 @@ set-hmc-version: yq
 	$(YQ) eval '.spec.version = "$(VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml
 	$(YQ) eval '.metadata.name = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml
 	$(YQ) eval '.spec.hmc.template = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml
+	$(YQ) eval '.metadata.name = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/templates/clustertemplatechain.yaml
 
 .PHONY: hmc-chart-release
 hmc-chart-release: set-hmc-version templates-generate ## Generate hmc helm chart
@@ -106,7 +107,10 @@ tidy:
 
 .PHONY: test
 test: generate-all fmt vet envtest tidy external-crd ## Run tests.
-	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e/scenarios) -coverprofile cover.out
+
+# E2E_CONFIG_B64 contains the configuration for e2e testing.
+E2E_CONFIG_B64 ?= ""
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling
 # compatibility with other vendors.
@@ -115,7 +119,8 @@ test-e2e: cli-install
 	@if [ "$$GINKGO_LABEL_FILTER" ]; then \
 		ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \
 	fi; \
-	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
+	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) E2E_CONFIG_B64=$(E2E_CONFIG_B64) \
+	go test ./test/e2e/scenarios/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint
@@ -324,6 +329,7 @@ dev-push: docker-build helm-push
 .PHONY: dev-templates
 dev-templates: templates-generate
 	$(KUBECTL) -n $(NAMESPACE) apply --force -f $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/templates
+	$(KUBECTL) -n $(NAMESPACE) apply --force -f $(PROVIDER_TEMPLATES_DIR)/hmc-templates/templates/clustertemplatechain.yaml
 
 .PHONY: dev-release
 dev-release:

docs/dev.md

@@ -166,7 +166,7 @@ GINKGO_LABEL_FILTER="provider:cloud" make test-e2e
 would run all cloud provider tests.  To see a list of all available labels run:
 
 ```bash
-ginkgo labels ./test/e2e
+ginkgo labels ./test/e2e/scenarios
 ```
 
 ### Nuke created resources

templates/provider/hmc-templates/templates/clustertemplatechain.yaml

@@ -0,0 +1,15 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ClusterTemplateChain
+metadata:
+  name: hmc-0-0-3
+  annotations:
+    helm.sh/resource-policy: keep
+spec:
+  supportedTemplates:
+    - name: aws-eks-0-0-2
+    - name: aws-hosted-cp-0-0-3
+    - name: aws-standalone-cp-0-0-3
+    - name: azure-hosted-cp-0-0-3
+    - name: azure-standalone-cp-0-0-3
+    - name: vsphere-hosted-cp-0-0-3
+    - name: vsphere-standalone-cp-0-0-3

test/e2e/config/config.go

@@ -0,0 +1,104 @@
+// Copyright 2024
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"gopkg.in/yaml.v3"
+
+	hmc "github.com/Mirantis/hmc/api/v1alpha1"
+	"github.com/Mirantis/hmc/test/e2e/templates"
+)
+
+type TestingProvider string
+
+const (
+	envVarE2EConfig = "E2E_CONFIG_B64"
+
+	TestingProviderAWS     TestingProvider = "aws"
+	TestingProviderAzure   TestingProvider = "azure"
+	TestingProviderVsphere TestingProvider = "vsphere"
+)
+
+var Config TestingConfig
+
+type TestingConfig = map[TestingProvider]ProviderTestingConfig
+
+type ProviderTestingConfig struct {
+	// Standalone contains the testing configuration for the standalone cluster deployment.
+	Standalone ClusterTestingConfig `yaml:"standalone,omitempty"`
+	// Standalone contains the testing configuration for the hosted cluster deployment.
+	Hosted ClusterTestingConfig `yaml:"hosted,omitempty"`
+}
+
+type ClusterTestingConfig struct {
+	// Upgrade is a boolean parameter that specifies whether the managed cluster upgrade should be tested.
+	Upgrade bool `yaml:"upgrade,omitempty"`
+	// Template is the name of the template to use when deploying a managed cluster.
+	// If unset:
+	// * The latest available template will be chosen
+	// * If upgrade is triggered, the latest available template with available upgrades will be chosen.
+	Template string `yaml:"template,omitempty"`
+	// UpgradeTemplate specifies the name of the template to upgrade to. Ignored if upgrade is set to false.
+	// If unset, the latest template available for the upgrade will be chosen.
+	UpgradeTemplate string `yaml:"upgradeTemplate,omitempty"`
+}
+
+func Parse() error {
+	decodedConfig, err := base64.StdEncoding.DecodeString(os.Getenv(envVarE2EConfig))
+	if err != nil {
+		return err
+	}
+	_, _ = fmt.Fprintf(GinkgoWriter, "E2e testing configuration:\n%s\n", decodedConfig)
+
+	err = yaml.Unmarshal(decodedConfig, &Config)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *ClusterTestingConfig) SetDefaults(clusterTemplates map[string][]hmc.AvailableUpgrade, templateType templates.Type) error {
+	var err error
+	if !c.Upgrade {
+		if c.Template == "" {
+			c.Template, err = templates.FindTemplate(clusterTemplates, templateType)
+			if err != nil {
+				return err
+			}
+		}
+		return templates.ValidateTemplate(clusterTemplates, c.Template)
+	}
+	if c.Template != "" && c.UpgradeTemplate != "" {
+		return templates.ValidateUpgradeSequence(clusterTemplates, c.Template, c.UpgradeTemplate)
+	}
+	c.Template, c.UpgradeTemplate, err = templates.FindTemplatesToUpgrade(clusterTemplates, templateType, c.Template)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *ProviderTestingConfig) String() string {
+	prettyConfig, err := yaml.Marshal(c)
+	Expect(err).NotTo(HaveOccurred())
+
+	return string(prettyConfig)
+}

test/e2e/kubeclient/kubeclient.go

@@ -21,23 +21,30 @@ import (
 	"os"
 	"path/filepath"
 
+	hcv2 "github.com/fluxcd/helm-controller/api/v2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	crclient "sigs.k8s.io/controller-runtime/pkg/client"
 
+	hmcmirantiscomv1alpha1 "github.com/Mirantis/hmc/api/v1alpha1"
 	"github.com/Mirantis/hmc/internal/utils/status"
 )
 
+var scheme = runtime.NewScheme()
+
 type KubeClient struct {
 	Client         kubernetes.Interface
+	CrClient       crclient.Client
 	ExtendedClient apiextensionsclientset.Interface
 	Config         *rest.Config
 
@@ -56,16 +63,16 @@ func NewFromLocal(namespace string) *KubeClient {
 // the kubeconfig from secret it needs an existing kubeclient.
 func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName string) *KubeClient {
 	GinkgoHelper()
-	return newKubeClient(kc.getKubeconfigSecretData(ctx, clusterName), namespace)
+	return newKubeClient(kc.getKubeconfigSecretData(ctx, namespace, clusterName), namespace)
 }
 
 // WriteKubeconfig writes the kubeconfig for the given clusterName to the
 // test/e2e directory returning the path to the file and a function to delete
 // it later.
-func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (string, func() error) {
+func (kc *KubeClient) WriteKubeconfig(ctx context.Context, namespace, clusterName string) (string, func() error) {
 	GinkgoHelper()
 
-	secretData := kc.getKubeconfigSecretData(ctx, clusterName)
+	secretData := kc.getKubeconfigSecretData(ctx, namespace, clusterName)
 
 	dir, err := os.Getwd()
 	Expect(err).NotTo(HaveOccurred())
@@ -89,11 +96,11 @@ func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (
 	return path, deleteFunc
 }
 
-func (kc *KubeClient) getKubeconfigSecretData(ctx context.Context, clusterName string) []byte {
+func (kc *KubeClient) getKubeconfigSecretData(ctx context.Context, namespace, clusterName string) []byte {
 	GinkgoHelper()
 
-	secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{})
-	Expect(err).NotTo(HaveOccurred(), "failed to get cluster: %q kubeconfig secret", clusterName)
+	secret, err := kc.Client.CoreV1().Secrets(namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{})
+	Expect(err).NotTo(HaveOccurred(), "failed to get cluster: %q kubeconfig secret in %s namespace", clusterName, namespace)
 
 	secretData, ok := secret.Data["value"]
 	Expect(ok).To(BeTrue(), "kubeconfig secret %q has no 'value' key", clusterName)
@@ -132,35 +139,45 @@ func newKubeClient(configBytes []byte, namespace string) *KubeClient {
 	clientSet, err := kubernetes.NewForConfig(config)
 	Expect(err).NotTo(HaveOccurred(), "failed to initialize kubernetes client")
 
+	err = hmcmirantiscomv1alpha1.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred(), "failed to add HMC API to scheme")
+
+	err = hcv2.AddToScheme(scheme)
+	Expect(err).NotTo(HaveOccurred(), "failed to add Flux helm controller API to scheme")
+
+	crClient, err := crclient.New(config, crclient.Options{Scheme: scheme})
+	Expect(err).NotTo(HaveOccurred(), "failed to create controller runtime client")
+
 	extendedClientSet, err := apiextensionsclientset.NewForConfig(config)
 	Expect(err).NotTo(HaveOccurred(), "failed to initialize apiextensions clientset")
 
 	return &KubeClient{
 		Namespace:      namespace,
 		Client:         clientSet,
+		CrClient:       crClient,
 		ExtendedClient: extendedClientSet,
 		Config:         config,
 	}
 }
 
 // GetDynamicClient returns a dynamic client for the given GroupVersionResource.
-func (kc *KubeClient) GetDynamicClient(gvr schema.GroupVersionResource, namespaced bool) dynamic.ResourceInterface { //nolint:revive
+func (kc *KubeClient) GetDynamicClient(gvr schema.GroupVersionResource, namespace string) dynamic.ResourceInterface {
 	GinkgoHelper()
 
 	client, err := dynamic.NewForConfig(kc.Config)
 	Expect(err).NotTo(HaveOccurred(), "failed to create dynamic client for resource: %s", gvr.String())
 
-	if !namespaced {
+	if namespace == "" {
 		return client.Resource(gvr)
 	}
 
-	return client.Resource(gvr).Namespace(kc.Namespace)
+	return client.Resource(gvr).Namespace(namespace)
 }
 
-func (kc *KubeClient) CreateOrUpdateUnstructuredObject(gvr schema.GroupVersionResource, obj *unstructured.Unstructured, namespaced bool) {
+func (kc *KubeClient) CreateOrUpdateUnstructuredObject(gvr schema.GroupVersionResource, obj *unstructured.Unstructured, namespace string) {
 	GinkgoHelper()
 
-	client := kc.GetDynamicClient(gvr, namespaced)
+	client := kc.GetDynamicClient(gvr, namespace)
 
 	kind, name := status.ObjKindName(obj)
 
@@ -181,18 +198,22 @@ func (kc *KubeClient) CreateOrUpdateUnstructuredObject(gvr schema.GroupVersionRe
 // namespace and returns a DeleteFunc to clean up the deployment.
 // The DeleteFunc is a no-op if the deployment has already been deleted.
 func (kc *KubeClient) CreateManagedCluster(
-	ctx context.Context, managedcluster *unstructured.Unstructured,
+	ctx context.Context, managedcluster *unstructured.Unstructured, namespace string,
 ) func() error {
 	GinkgoHelper()
 
 	kind := managedcluster.GetKind()
 	Expect(kind).To(Equal("ManagedCluster"))
 
+	if namespace != "" {
+		managedcluster.SetNamespace(namespace)
+	}
+
 	client := kc.GetDynamicClient(schema.GroupVersionResource{
 		Group:    "hmc.mirantis.com",
 		Version:  "v1alpha1",
 		Resource: "managedclusters",
-	}, true)
+	}, namespace)
 
 	_, err := client.Create(ctx, managedcluster, metav1.CreateOptions{})
 	if !apierrors.IsAlreadyExists(err) {
@@ -209,14 +230,14 @@ func (kc *KubeClient) CreateManagedCluster(
 }
 
 // GetCluster returns a Cluster resource by name.
-func (kc *KubeClient) GetCluster(ctx context.Context, clusterName string) (*unstructured.Unstructured, error) {
+func (kc *KubeClient) GetCluster(ctx context.Context, namespace, clusterName string) (*unstructured.Unstructured, error) {
 	gvr := schema.GroupVersionResource{
 		Group:    "cluster.x-k8s.io",
 		Version:  "v1beta1",
 		Resource: "clusters",
 	}
 
-	client := kc.GetDynamicClient(gvr, true)
+	client := kc.GetDynamicClient(gvr, namespace)
 
 	cluster, err := client.Get(ctx, clusterName, metav1.GetOptions{})
 	if err != nil {
@@ -229,9 +250,9 @@ func (kc *KubeClient) GetCluster(ctx context.Context, clusterName string) (*unst
 // listResource returns a list of resources for the given GroupVersionResource
 // affiliated with the given clusterName.
 func (kc *KubeClient) listResource(
-	ctx context.Context, gvr schema.GroupVersionResource, clusterName string,
+	ctx context.Context, gvr schema.GroupVersionResource, namespace, clusterName string,
 ) ([]unstructured.Unstructured, error) {
-	client := kc.GetDynamicClient(gvr, true)
+	client := kc.GetDynamicClient(gvr, namespace)
 
 	resources, err := client.List(ctx, metav1.ListOptions{
 		LabelSelector: "cluster.x-k8s.io/cluster-name=" + clusterName,
@@ -244,38 +265,38 @@ func (kc *KubeClient) listResource(
 }
 
 // ListMachines returns a list of Machine resources for the given cluster.
-func (kc *KubeClient) ListMachines(ctx context.Context, clusterName string) ([]unstructured.Unstructured, error) {
+func (kc *KubeClient) ListMachines(ctx context.Context, namespace, clusterName string) ([]unstructured.Unstructured, error) {
 	GinkgoHelper()
 
 	return kc.listResource(ctx, schema.GroupVersionResource{
 		Group:    "cluster.x-k8s.io",
 		Version:  "v1beta1",
 		Resource: "machines",
-	}, clusterName)
+	}, namespace, clusterName)
 }
 
 // ListMachineDeployments returns a list of MachineDeployment resources for the
 // given cluster.
 func (kc *KubeClient) ListMachineDeployments(
-	ctx context.Context, clusterName string,
+	ctx context.Context, namespace, clusterName string,
 ) ([]unstructured.Unstructured, error) {
 	GinkgoHelper()
 
 	return kc.listResource(ctx, schema.GroupVersionResource{
 		Group:    "cluster.x-k8s.io",
 		Version:  "v1beta1",
 		Resource: "machinedeployments",
-	}, clusterName)
+	}, namespace, clusterName)
 }
 
 func (kc *KubeClient) ListK0sControlPlanes(
-	ctx context.Context, clusterName string,
+	ctx context.Context, namespace, clusterName string,
 ) ([]unstructured.Unstructured, error) {
 	GinkgoHelper()
 
 	return kc.listResource(ctx, schema.GroupVersionResource{
 		Group:    "controlplane.cluster.x-k8s.io",
 		Version:  "v1beta1",
 		Resource: "k0scontrolplanes",
-	}, clusterName)
+	}, namespace, clusterName)
 }

test/e2e/managedcluster/aws/aws.go

@@ -39,7 +39,7 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient,
 		Group:    "infrastructure.cluster.x-k8s.io",
 		Version:  "v1beta2",
 		Resource: "awsclusters",
-	}, true)
+	}, managedcluster.Namespace)
 
 	awsCluster, err := c.Get(ctx, clusterName, metav1.GetOptions{})
 	Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster")