From 9fe8b76fb21f0f3864ff17d9385e9f25e6b281c7 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 27 Nov 2024 12:56:33 -0800 Subject: [PATCH 01/23] Fix e2e tests and add more logging --- Makefile | 31 ++++++++++-- test/e2e/e2e_suite_test.go | 12 +++++ test/e2e/kubeclient/kubeclient.go | 15 ++++++ .../clusteridentity/clusteridentity.go | 39 ++++++++++++++- test/e2e/managedcluster/managedcluster.go | 47 +++++++++++++++++-- .../resources/aws-hosted-cp.yaml.tpl | 2 +- .../resources/aws-standalone-cp.yaml.tpl | 4 +- .../resources/azure-hosted-cp.yaml.tpl | 2 +- .../resources/azure-standalone-cp.yaml.tpl | 2 +- .../resources/vsphere-hosted-cp.yaml.tpl | 2 +- .../resources/vsphere-standalone-cp.yaml.tpl | 2 +- test/e2e/provider_aws_test.go | 4 ++ test/e2e/provider_azure_test.go | 4 ++ test/utils/utils.go | 9 +++- 14 files changed, 158 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 561d85d38..6ab86dc5a 100644 --- a/Makefile +++ b/Makefile @@ -72,8 +72,33 @@ set-hmc-version: yq $(YQ) eval '.metadata.name = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml $(YQ) eval '.spec.hmc.template = "hmc-$(FQDN_VERSION)"' -i $(PROVIDER_TEMPLATES_DIR)/hmc-templates/files/release.yaml +.PHONY: test-hmc-version +test-hmc-version: yq + @TEMPLATE_DIR=templates/provider/hmc-templates/files/templates/*; \ + cutversion=$$(echo $(VERSION) | sed "s/.*-//g"); \ + echo "cutversion = $cutversion"; \ + for template in $$TEMPLATE_DIR; do \ + if [ $$($(YQ) eval '.kind' $$template) = 'ClusterTemplate' ]; then \ + if grep --quiet "chartVersion:.*-"$$cutversion $$template; then \ + echo "skipping already processed templates $$template"; \ + else \ + echo $$template; \ + sed -i "s/chartVersion:.*[^$$cutversion].*/&-$$cutversion/g" "$$template"; \ + fi \ + fi; \ + done; \ + CHART_DIR=templates/cluster/**/Chart.yaml; \ + for chart in $$CHART_DIR; do \ + if grep --quiet "version:.*-$$cutversion" $$chart; then \ + echo "skipping already processed templates $$chart"; \ + else \ + echo $$chart; \ + sed -i "s/version:.*[^$$cutversion]*./&-$$cutversion/g" "$$chart"; \ + fi \ + done + .PHONY: hmc-chart-release -hmc-chart-release: set-hmc-version templates-generate ## Generate hmc helm chart +hmc-chart-release: set-hmc-version test-hmc-version templates-generate ## Generate hmc helm chart .PHONY: hmc-dist-release hmc-dist-release: $(HELM) $(YQ) @@ -110,7 +135,7 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests. # Utilize Kind or modify the e2e tests to load the image locally, enabling # compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster. +.PHONY: test-e2e test-hmc-version# Run the e2e tests using a Kind k8s instance as the management cluster. test-e2e: cli-install @if [ "$$GINKGO_LABEL_FILTER" ]; then \ ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \ @@ -352,7 +377,7 @@ dev-eks-creds: dev-aws-creds dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates dev-release .PHONY: test-apply -test-apply: set-hmc-version helm-package dev-deploy dev-templates dev-release +test-apply: set-hmc-version test-hmc-version helm-package dev-deploy dev-templates dev-release .PHONY: dev-destroy dev-destroy: kind-undeploy registry-undeploy ## Destroy the development environment by deleting the kind cluster and local registry. diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index e76a4c245..d67ccc546 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -66,6 +66,15 @@ var _ = BeforeSuite(func() { } return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + + Eventually(func() error { + err = managedcluster.ValidateClusterTemplates(context.Background(), kc) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "Controller validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) }) var _ = AfterSuite(func() { @@ -93,6 +102,9 @@ func verifyControllersUp(kc *kubeclient.KubeClient) error { managedcluster.ProviderAWS, managedcluster.ProviderAzure, managedcluster.ProviderVSphere, + managedcluster.ProviderK0smotron, + managedcluster.ProviderK0smotronBootstrap, + managedcluster.ProviderK0smotronControlPlane, } for _, provider := range providers { diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go index e3801e4e0..b1dfc8fce 100644 --- a/test/e2e/kubeclient/kubeclient.go +++ b/test/e2e/kubeclient/kubeclient.go @@ -243,6 +243,21 @@ func (kc *KubeClient) listResource( return resources.Items, nil } +func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) { + client := kc.GetDynamicClient(schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "clustertemplates", + }, true) + + resources, err := client.List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, fmt.Errorf("failed to list cluster templates") + } + + return resources.Items, nil +} + // ListMachines returns a list of Machine resources for the given cluster. func (kc *KubeClient) ListMachines(ctx context.Context, clusterName string) ([]unstructured.Unstructured, error) { GinkgoHelper() diff --git a/test/e2e/managedcluster/clusteridentity/clusteridentity.go b/test/e2e/managedcluster/clusteridentity/clusteridentity.go index 41e01c77e..bc05348df 100644 --- a/test/e2e/managedcluster/clusteridentity/clusteridentity.go +++ b/test/e2e/managedcluster/clusteridentity/clusteridentity.go @@ -134,7 +134,7 @@ func New(kc *kubeclient.KubeClient, provider managedcluster.ProviderType) *Clust ci.createSecret(kc) ci.createClusterIdentity(kc) ci.createCredential(kc) - + ci.waitForCredentialReady(kc) return &ci } @@ -252,3 +252,40 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) { kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced) } + +func (ci *ClusterIdentity) waitForCredentialReady(kc *kubeclient.KubeClient) { + GinkgoHelper() + + By(fmt.Sprintf("waiting for %s credential to be ready", ci.Kind)) + + ctx := context.Background() + credName := fmt.Sprintf("%s-cred", ci.IdentityName) + + gvr := schema.GroupVersionResource{ + Group: "hmc.mirantis.com", + Version: "v1alpha1", + Resource: "credentials", + } + + Eventually(func() error { + creds, err := kc.GetDynamicClient(gvr, true).Get(ctx, credName, metav1.GetOptions{}) + if err != nil { + return err + } + + ready, found, err := unstructured.NestedBool(creds.Object, "status", "ready") + if err != nil { + return err + } + + if !found { + return fmt.Errorf("ready status not found on credential %s", credName) + } + + if !ready { + return fmt.Errorf("credential %s not ready", credName) + } + + return nil + }).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed()) +} diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index 72efa96f8..fbee92d44 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -15,6 +15,7 @@ package managedcluster import ( + "context" _ "embed" "fmt" "os" @@ -26,17 +27,22 @@ import ( . "github.com/onsi/gomega" "gopkg.in/yaml.v3" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/utils/env" + "github.com/Mirantis/hmc/test/e2e/kubeclient" "github.com/Mirantis/hmc/test/utils" ) type ProviderType string const ( - ProviderCAPI ProviderType = "cluster-api" - ProviderAWS ProviderType = "infrastructure-aws" - ProviderAzure ProviderType = "infrastructure-azure" - ProviderVSphere ProviderType = "infrastructure-vsphere" + ProviderCAPI ProviderType = "cluster-api" + ProviderAWS ProviderType = "infrastructure-aws" + ProviderAzure ProviderType = "infrastructure-azure" + ProviderVSphere ProviderType = "infrastructure-vsphere" + ProviderK0smotron ProviderType = "infrastructure-k0sproject-k0smotron" + ProviderK0smotronBootstrap ProviderType = "bootstrap-k0sproject-k0smotron" + ProviderK0smotronControlPlane ProviderType = "control-plane-k0sproject-k0smotron" providerLabel = "cluster.x-k8s.io/provider" ) @@ -138,17 +144,48 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { Fail(fmt.Sprintf("Unsupported template: %s", templateName)) } + version := env.GetString("VERSION", "") + index := strings.LastIndex(env.GetString("VERSION", ""), "-") + if index > 0 { + version = version[index:] + } + GinkgoT().Setenv("BUILD_VERSION", version) + managedClusterConfigBytes, err := envsubst.Bytes(managedClusterTemplateBytes) Expect(err).NotTo(HaveOccurred(), "failed to substitute environment variables") var managedClusterConfig map[string]any - + By(fmt.Sprintf("Cluster being applied\n %s", managedClusterConfigBytes)) err = yaml.Unmarshal(managedClusterConfigBytes, &managedClusterConfig) Expect(err).NotTo(HaveOccurred(), "failed to unmarshal deployment config") return &unstructured.Unstructured{Object: managedClusterConfig} } +func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error { + templates, err := client.ListClusterTemplates(ctx) + if err != nil { + return fmt.Errorf("failed to list cluster templates: %w", err) + } + + for _, template := range templates { + valid, found, err := unstructured.NestedBool(template.Object, "status", "valid") + if err != nil { + return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err) + } + + if !found { + return fmt.Errorf("valid flag for template %s not found", template.GetName()) + } + + if !valid { + return fmt.Errorf("template %s is still invalid", template.GetName()) + } + } + + return nil +} + func ValidateDeploymentVars(v []string) { GinkgoHelper() diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index 8a2700c63..8f5a84885 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-hosted-cp-0-0-3 + template: aws-hosted-cp-0-0-3${BUILD_VERSION} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: diff --git a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl index 24c449bc0..3a42d828e 100644 --- a/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: aws-standalone-cp-0-0-3 + template: aws-standalone-cp-0-0-3${BUILD_VERSION} credential: ${AWS_CLUSTER_IDENTITY}-cred config: clusterIdentity: @@ -15,5 +15,7 @@ spec: workersNumber: ${WORKERS_NUMBER:=1} controlPlane: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 32 worker: instanceType: ${AWS_INSTANCE_TYPE:=t3.small} + rootVolumeSize: 32 diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl index 76da17cbb..407096ab2 100644 --- a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-hosted-cp-0-0-3 + template: azure-hosted-cp-0-0-3${BUILD_VERSION} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: location: "${AZURE_REGION}" diff --git a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl index 3894b7b39..14e5bce67 100644 --- a/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-standalone-cp.yaml.tpl @@ -4,7 +4,7 @@ metadata: name: ${MANAGED_CLUSTER_NAME} namespace: ${NAMESPACE} spec: - template: azure-standalone-cp-0-0-3 + template: azure-standalone-cp-0-0-3${BUILD_VERSION} credential: ${AZURE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: 1 diff --git a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl index c0475f3f4..3d2352e71 100644 --- a/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-hosted-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-hosted-cp-0-0-3 + template: vsphere-hosted-cp-0-0-3${BUILD_VERSION} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl index cc5fa87b3..90cbe942f 100644 --- a/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/vsphere-standalone-cp.yaml.tpl @@ -3,7 +3,7 @@ kind: ManagedCluster metadata: name: ${MANAGED_CLUSTER_NAME} spec: - template: vsphere-standalone-cp-0-0-3 + template: vsphere-standalone-cp-0-0-3${BUILD_VERSION} credential: ${VSPHERE_CLUSTER_IDENTITY}-cred config: controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1} diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 6614698b4..79dffd0e0 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -124,6 +124,10 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + return managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index fcdbe27c7..8097084e1 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -123,6 +123,10 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + return managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Create azure credential secret") clusteridentity.New(standaloneClient, managedcluster.ProviderAzure) diff --git a/test/utils/utils.go b/test/utils/utils.go index 56ce5ee33..507617843 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -38,12 +38,17 @@ func Run(cmd *exec.Cmd) ([]byte, error) { command := prepareCmd(cmd) _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) - output, err := cmd.Output() + cmd.Stdout = GinkgoWriter + cmd.Stderr = GinkgoWriter + + err := cmd.Run() if err != nil { return nil, handleCmdError(err, command) } - return output, nil + _, _ = fmt.Fprintf(GinkgoWriter, "done running: %s\n", command) + + return nil, nil } func handleCmdError(err error, command string) error { From 3e9c97429ef51faee72b54a759bdd86590b93f09 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 27 Nov 2024 15:17:49 -0800 Subject: [PATCH 02/23] fix up logging, ensure we get a public subnet for the hosted deploy on aws --- test/e2e/e2e_suite_test.go | 2 +- test/e2e/managedcluster/aws/aws.go | 24 ++++++++++++------- .../resources/aws-hosted-cp.yaml.tpl | 1 + test/e2e/provider_aws_test.go | 13 +++++++--- test/e2e/provider_azure_test.go | 13 +++++++--- 5 files changed, 38 insertions(+), 15 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index d67ccc546..b1484713d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -70,7 +70,7 @@ var _ = BeforeSuite(func() { Eventually(func() error { err = managedcluster.ValidateClusterTemplates(context.Background(), kc) if err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "Controller validation failed: %v\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) return err } return nil diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 441cfc499..8f4454cba 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -52,14 +52,23 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - subnet, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue(), "failed to cast subnet to map") + // find the first public subnet + for _, s := range subnets { + subnet, ok := s.(map[string]any) + Expect(ok).To(BeTrue(), "failed to cast subnet to map") - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") + if isPublic, ok := subnet["isPublic"].(bool); ok && isPublic { + subnetID, ok := subnet["resourceID"].(string) + Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + subnetAZ, ok := subnet["availabilityZone"].(string) + Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") + + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetID, subnetID) + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetAvailabilityZone, subnetAZ) + break + } + } securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") @@ -67,7 +76,6 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(managedcluster.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetAvailabilityZone, subnetAZ) + GinkgoT().Setenv(managedcluster.EnvVarAWSSecurityGroupID, securityGroupID) } diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index 8f5a84885..e76560743 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -14,6 +14,7 @@ spec: subnets: - id: ${AWS_SUBNET_ID} availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} + isPublic: true instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 79dffd0e0..b62aac1e4 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/utils/env" internalutils "github.com/Mirantis/hmc/internal/utils" "github.com/Mirantis/hmc/test/e2e/kubeclient" @@ -106,7 +107,7 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order kubeCfgPath, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), clusterName) GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", "test-apply") + cmd := exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) @@ -125,8 +126,14 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) Eventually(func() error { - return managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) - }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Ensure cluster templates valid") + err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index 8097084e1..cac559336 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -23,6 +23,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/utils/env" internalutils "github.com/Mirantis/hmc/internal/utils" "github.com/Mirantis/hmc/test/e2e/kubeclient" @@ -107,7 +108,7 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("Deploy onto standalone cluster") GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", "test-apply") + cmd := exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) @@ -124,8 +125,14 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) Eventually(func() error { - return managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) - }).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + By("Ensure cluster templates valid") + err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) By("Create azure credential secret") clusteridentity.New(standaloneClient, managedcluster.ProviderAzure) From 903682efaf74f208d54d4154d271d398b51f5fd7 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 27 Nov 2024 16:29:50 -0800 Subject: [PATCH 03/23] Add version parameter to test-apply call --- test/e2e/e2e_suite_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b1484713d..66c4bb888 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -18,6 +18,7 @@ import ( "bufio" "context" "fmt" + "k8s.io/utils/env" "net/url" "os" "os/exec" @@ -48,11 +49,11 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace) - By("building and deploying the controller-manager") + By(fmt.Sprintf("building and deploying the controller-manager - Version: %s", env.GetString("VERSION", ""))) cmd := exec.Command("make", "kind-deploy") _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) - cmd = exec.Command("make", "test-apply") + exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) From aaa5b1d2dd87f7fac2dc48e578c160d6054d7a18 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 28 Nov 2024 09:20:20 -0800 Subject: [PATCH 04/23] fix linting --- test/e2e/e2e_suite_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 66c4bb888..b055786e8 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -18,7 +18,6 @@ import ( "bufio" "context" "fmt" - "k8s.io/utils/env" "net/url" "os" "os/exec" @@ -31,6 +30,7 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/env" "k8s.io/utils/ptr" internalutils "github.com/Mirantis/hmc/internal/utils" From 4942d50a52ea7a9c7ee7d4a63e6059607993da74 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 28 Nov 2024 09:45:11 -0800 Subject: [PATCH 05/23] Add merge_commit_sha to build outputs step --- .github/workflows/build_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index fe2fa81cd..20de44cf3 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -31,6 +31,7 @@ jobs: version: ${{ steps.vars.outputs.version }} clustername: ${{ steps.vars.outputs.clustername }} pr: ${{ steps.pr.outputs.result }} + merge_commit_sha: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}} steps: - name: Get PR ref uses: actions/github-script@v7 From da9f5f55cc34f616d006591adfdc4383f02edda1 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 28 Nov 2024 09:55:21 -0800 Subject: [PATCH 06/23] For testing only, run a duplicate of the regular workflow --- .github/workflows/build_test.yml | 1 - .github/workflows/test_e2e.yml | 263 +++++++++++++++++++++++++++++++ test/e2e/e2e_suite_test.go | 8 +- test/e2e/provider_aws_test.go | 5 +- test/e2e/provider_azure_test.go | 5 +- test/utils/utils.go | 9 +- 6 files changed, 276 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test_e2e.yml diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml index 20de44cf3..fe2fa81cd 100644 --- a/.github/workflows/build_test.yml +++ b/.github/workflows/build_test.yml @@ -31,7 +31,6 @@ jobs: version: ${{ steps.vars.outputs.version }} clustername: ${{ steps.vars.outputs.clustername }} pr: ${{ steps.pr.outputs.result }} - merge_commit_sha: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}} steps: - name: Get PR ref uses: actions/github-script@v7 diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml new file mode 100644 index 000000000..69ed6ffc9 --- /dev/null +++ b/.github/workflows/test_e2e.yml @@ -0,0 +1,263 @@ +name: CI +on: + pull_request: + types: + - labeled + - opened + - synchronize + - reopened + paths-ignore: + - 'config/**' + - '**.md' + push: + tags: + - '*' + +env: + GO_VERSION: '1.22' + REGISTRY_REPO: 'oci://ghcr.io/mirantis/hmc/charts-ci' + +jobs: + build: + concurrency: + group: build-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + name: Build and Unit Test + runs-on: ubuntu-latest + outputs: + version: ${{ steps.vars.outputs.version }} + clustername: ${{ steps.vars.outputs.clustername }} + pr: ${{ steps.pr.outputs.result }} + steps: + - name: Get PR ref + uses: actions/github-script@v7 + id: pr + with: + script: | + const { data: pullRequest } = await github.rest.pulls.get({ + ...context.repo, + pull_number: context.payload.pull_request.number, + }); + return pullRequest + - name: Checkout repository + uses: actions/checkout@v4 + with: + ref: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}} + fetch-depth: 0 + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + - name: Lint + run: GOLANGCI_LINT_TIMEOUT=10m make lint + - name: Verify all generated pieces are up-to-date + run: make generate-all && git add -N . && git diff --exit-code + - name: Unit tests + run: | + make test + - name: Set up Buildx + uses: docker/setup-buildx-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Get outputs + id: vars + run: | + GIT_VERSION=$(git describe --tags --always) + echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT + echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT + - name: Build and push HMC controller image + uses: docker/build-push-action@v6 + with: + build-args: | + LD_FLAGS=-s -w -X github.com/Mirantis/hmc/internal/build.Version=${{ steps.vars.outputs.version }} + context: . + platforms: linux/amd64 + tags: | + ghcr.io/mirantis/hmc/controller-ci:${{ steps.vars.outputs.version }} + push: true + cache-from: type=gha + cache-to: type=gha,mode=max + - name: Prepare and push HMC template charts + run: | + make hmc-chart-release + make helm-push + + controller-e2etest: + name: E2E Controller + runs-on: ubuntu-latest + needs: build + concurrency: + group: controller-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'controller' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cloud-e2etest-logs + path: | + test/e2e/*.log + + provider-cloud-e2etest: + name: E2E Cloud Providers + runs-on: ubuntu-latest + if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }} + needs: build + concurrency: + group: cloud-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 + AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - uses: actions/checkout@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'provider:cloud' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: cloud-e2etest-logs + path: | + test/e2e/*.log + + provider-onprem-e2etest: + name: E2E On-Prem Providers + runs-on: self-hosted + if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }} + needs: build + concurrency: + group: onprem-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + env: + VSPHERE_USER: ${{ secrets.CI_VSPHERE_USER }} + VSPHERE_PASSWORD: ${{ secrets.CI_VSPHERE_PASSWORD }} + VSPHERE_SERVER: ${{ secrets.CI_VSPHERE_SERVER }} + VSPHERE_THUMBPRINT: ${{ secrets.CI_VSPHERE_THUMBPRINT }} + VSPHERE_DATACENTER: ${{ secrets.CI_VSPHERE_DATACENTER }} + VSPHERE_DATASTORE: ${{ secrets.CI_VSPHERE_DATASTORE }} + VSPHERE_RESOURCEPOOL: ${{ secrets.CI_VSPHERE_RESOURCEPOOL }} + VSPHERE_FOLDER: ${{ secrets.CI_VSPHERE_FOLDER }} + VSPHERE_CONTROL_PLANE_ENDPOINT: ${{ secrets.CI_VSPHERE_CONTROL_PLANE_ENDPOINT }} + VSPHERE_VM_TEMPLATE: ${{ secrets.CI_VSPHERE_VM_TEMPLATE }} + VSPHERE_NETWORK: ${{ secrets.CI_VSPHERE_NETWORK }} + VSPHERE_SSH_KEY: ${{ secrets.CI_VSPHERE_SSH_KEY }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Setup kubectl + uses: azure/setup-kubectl@v4 + - name: Run E2E tests + env: + GINKGO_LABEL_FILTER: 'provider:onprem' + MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }} + IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}' + VERSION: ${{ needs.build.outputs.version }} + run: | + make test-e2e + - name: Archive test results + if: ${{ failure() }} + uses: actions/upload-artifact@v4 + with: + name: onprem-e2etest-logs + path: | + test/e2e/*.log + + cleanup: + name: Cleanup + needs: + - build + - provider-cloud-e2etest + runs-on: ubuntu-latest + if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }} + timeout-minutes: 15 + outputs: + clustername: ${{ needs.build.outputs.clustername }} + version: ${{ needs.build.outputs.version }} + pr: ${{ needs.build.outputs.pr }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}} + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: AWS Test Resources + env: + AWS_REGION: us-west-2 + AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }} + AZURE_REGION: westus2 + AZURE_SUBSCRIPTION_ID: ${{ secrets.CI_AZURE_SUBSCRIPTION_ID }} + AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }} + AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }} + CLUSTER_NAME: '${{ needs.build.outputs.clustername }}' + run: | + make dev-aws-nuke + make dev-azure-nuke diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index b055786e8..5a4843f64 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -51,10 +51,12 @@ var _ = BeforeSuite(func() { By(fmt.Sprintf("building and deploying the controller-manager - Version: %s", env.GetString("VERSION", ""))) cmd := exec.Command("make", "kind-deploy") - _, err := utils.Run(cmd) + output, err := utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) - exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") - _, err = utils.Run(cmd) + cmd = exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") + output, err = utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) By("validating that the hmc-controller and CAPI provider controllers are running and ready") diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index b62aac1e4..52586eba3 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -107,8 +107,9 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order kubeCfgPath, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), clusterName) GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") - _, err := utils.Run(cmd) + cmd := exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") + output, err := utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) diff --git a/test/e2e/provider_azure_test.go b/test/e2e/provider_azure_test.go index cac559336..a3b5fd2c3 100644 --- a/test/e2e/provider_azure_test.go +++ b/test/e2e/provider_azure_test.go @@ -108,8 +108,9 @@ var _ = Context("Azure Templates", Label("provider:cloud", "provider:azure"), Or By("Deploy onto standalone cluster") GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", "VERSION=", env.GetString("VERSION", ""), "test-apply") - _, err := utils.Run(cmd) + cmd := exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") + output, err := utils.Run(cmd) + _, _ = fmt.Fprint(GinkgoWriter, string(output)) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) diff --git a/test/utils/utils.go b/test/utils/utils.go index 507617843..56ce5ee33 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -38,17 +38,12 @@ func Run(cmd *exec.Cmd) ([]byte, error) { command := prepareCmd(cmd) _, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command) - cmd.Stdout = GinkgoWriter - cmd.Stderr = GinkgoWriter - - err := cmd.Run() + output, err := cmd.Output() if err != nil { return nil, handleCmdError(err, command) } - _, _ = fmt.Fprintf(GinkgoWriter, "done running: %s\n", command) - - return nil, nil + return output, nil } func handleCmdError(err error, command string) error { From ea7b8e1f958c2edb274dd77461a93bcbb3967d20 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 28 Nov 2024 12:34:19 -0800 Subject: [PATCH 07/23] Formatting change --- test/e2e/e2e_suite_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 5a4843f64..45290570d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -49,7 +49,8 @@ func TestE2E(t *testing.T) { var _ = BeforeSuite(func() { GinkgoT().Setenv(managedcluster.EnvVarNamespace, internalutils.DefaultSystemNamespace) - By(fmt.Sprintf("building and deploying the controller-manager - Version: %s", env.GetString("VERSION", ""))) + By(fmt.Sprintf("building and deploying the controller-manager - Version: %s", + env.GetString("VERSION", ""))) cmd := exec.Command("make", "kind-deploy") output, err := utils.Run(cmd) _, _ = fmt.Fprint(GinkgoWriter, string(output)) From 0378b05fd7fef48d8ddcacadeda59b07f433e2d8 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Thu, 28 Nov 2024 15:11:53 -0800 Subject: [PATCH 08/23] Add tmate to help debug --- .github/workflows/test_e2e.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 69ed6ffc9..5eefbfb62 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -144,6 +144,11 @@ jobs: AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }} AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }} steps: + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + with: + detached: true + timeout-minutes: 90 - name: Checkout repository uses: actions/checkout@v4 with: From 167746321bb48cd7907a3829774b7f77c5c2d5b6 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Fri, 29 Nov 2024 10:24:54 -0800 Subject: [PATCH 09/23] Add force push of templates --- .github/workflows/test_e2e.yml | 1 + Makefile | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_e2e.yml b/.github/workflows/test_e2e.yml index 5eefbfb62..698edf891 100644 --- a/.github/workflows/test_e2e.yml +++ b/.github/workflows/test_e2e.yml @@ -86,6 +86,7 @@ jobs: run: | make hmc-chart-release make helm-push + make FORCE_PUSH=true test-hmc-version helm-push controller-e2etest: name: E2E Controller diff --git a/Makefile b/Makefile index 6ab86dc5a..316361a4b 100644 --- a/Makefile +++ b/Makefile @@ -322,7 +322,7 @@ helm-push: helm-package else \ chart_exists=$$($(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \ fi; \ - if [ -z "$$chart_exists" ]; then \ + if [ -z "$$chart_exists" ] && [ -z $$FORCE_PUSH ]; then \ echo "Chart $$chart_name version $$chart_version already exists in the repository."; \ else \ if $(REGISTRY_IS_OCI); then \ From d99663f41100266d89d2b40b9fdfac63f197e3ed Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Fri, 29 Nov 2024 12:27:25 -0800 Subject: [PATCH 10/23] Ensure the subnet name is set --- test/e2e/managedcluster/azure/azure.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/e2e/managedcluster/azure/azure.go b/test/e2e/managedcluster/azure/azure.go index 2880badcf..ed53e6f5f 100644 --- a/test/e2e/managedcluster/azure/azure.go +++ b/test/e2e/managedcluster/azure/azure.go @@ -76,8 +76,23 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) resourceGroup := spec["resourceGroup"] GinkgoT().Setenv("AZURE_RESOURCE_GROUP", fmt.Sprintf("%s", resourceGroup)) - subnetMap, ok := subnets[0].(map[string]any) - Expect(ok).To(BeTrue()) + + var subnetMap map[string]any + for _, subnet := range subnets { + sMap, ok := subnet.(map[string]any) + Expect(ok).To(BeTrue()) + + routeTable, exists, err := unstructured.NestedMap(sMap, "routeTable") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) + routeTableName := routeTable["name"] + + if routeTableName != nil && len(fmt.Sprintf("%s", routeTableName)) > 0 { + subnetMap = sMap + break + } + } + subnetName := subnetMap["name"] GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) From eabb23e364c78001c0e3ea938c424b2c99863844 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Fri, 29 Nov 2024 14:43:42 -0800 Subject: [PATCH 11/23] Add all AWS subnets to hosted deploy --- test/e2e/managedcluster/aws/aws.go | 26 +++++++++---------- test/e2e/managedcluster/constants.go | 2 +- .../resources/aws-hosted-cp.yaml.tpl | 4 +-- 3 files changed, 14 insertions(+), 18 deletions(-) diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 8f4454cba..0c27239cb 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -52,30 +53,27 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster subnets") Expect(found).To(BeTrue(), "AWS cluster has no subnets") - // find the first public subnet - for _, s := range subnets { + type awsSubnetMaps []map[string]any + subnetMaps := make(awsSubnetMaps, len(subnets)) + for i, s := range subnets { subnet, ok := s.(map[string]any) Expect(ok).To(BeTrue(), "failed to cast subnet to map") - - if isPublic, ok := subnet["isPublic"].(bool); ok && isPublic { - subnetID, ok := subnet["resourceID"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet ID to string") - - subnetAZ, ok := subnet["availabilityZone"].(string) - Expect(ok).To(BeTrue(), "failed to cast subnet availability zone to string") - - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetID, subnetID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnetAvailabilityZone, subnetAZ) - break + subnetMaps[i] = map[string]any{ + "isPublic": subnet["isPublic"], + "availabilityZone": subnet["availabilityZone"], + "id": subnet["resourceID"], } } + subnetYaml, err := yaml.Marshal(subnetMaps) + Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") + GinkgoT().Setenv(managedcluster.EnvVarSubnets, string(subnetYaml)) + securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") Expect(err).NotTo(HaveOccurred(), "failed to get AWS cluster security group ID") Expect(found).To(BeTrue(), "AWS cluster has no security group ID") GinkgoT().Setenv(managedcluster.EnvVarAWSVPCID, vpcID) - GinkgoT().Setenv(managedcluster.EnvVarAWSSecurityGroupID, securityGroupID) } diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go index 4f18a7832..5100c5449 100644 --- a/test/e2e/managedcluster/constants.go +++ b/test/e2e/managedcluster/constants.go @@ -33,7 +33,7 @@ const ( EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" EnvVarAWSSecurityGroupID = "AWS_SG_ID" EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarPublicIP = "AWS_PUBLIC_IP" + EnvVarSubnets = "AWS_SUBNETS" // VSphere EnvVarVSphereUser = "VSPHERE_USER" diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index e76560743..feac07edf 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -12,9 +12,7 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - - id: ${AWS_SUBNET_ID} - availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE} - isPublic: true + ${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} From 1044e80ea713edef033c517fdb09ef8c8c15fa49 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Fri, 29 Nov 2024 15:16:15 -0800 Subject: [PATCH 12/23] Fix up AWS validation, update Azure template as a test --- .../azure-hosted-cp/templates/azurecluster.yaml | 3 ++- test/e2e/managedcluster/aws/aws.go | 2 +- test/e2e/managedcluster/constants.go | 16 +++++++--------- test/e2e/managedcluster/managedcluster.go | 4 +--- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index 3ec9a7a22..2bdb10501 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -17,7 +17,8 @@ spec: name: {{ .Values.network.vnetName }} subnets: - name: {{ .Values.network.nodeSubnetName }} - role: node + # TODO this is just a test, if this works we need to consider changing this to be able to specify multiple subnets + role: cluster routeTable: name: {{ .Values.network.routeTableName }} securityGroup: diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 0c27239cb..c655f54ca 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -67,7 +67,7 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, subnetYaml, err := yaml.Marshal(subnetMaps) Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") - GinkgoT().Setenv(managedcluster.EnvVarSubnets, string(subnetYaml)) + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, string(subnetYaml)) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") diff --git a/test/e2e/managedcluster/constants.go b/test/e2e/managedcluster/constants.go index 5100c5449..1d5d2104f 100644 --- a/test/e2e/managedcluster/constants.go +++ b/test/e2e/managedcluster/constants.go @@ -25,15 +25,13 @@ const ( EnvVarNoCleanup = "NO_CLEANUP" // AWS - EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" - EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" - EnvVarAWSVPCID = "AWS_VPC_ID" - EnvVarAWSSubnetID = "AWS_SUBNET_ID" - EnvVarAWSSubnetAvailabilityZone = "AWS_SUBNET_AVAILABILITY_ZONE" - EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" - EnvVarAWSSecurityGroupID = "AWS_SG_ID" - EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" - EnvVarSubnets = "AWS_SUBNETS" + EnvVarAWSAccessKeyID = "AWS_ACCESS_KEY_ID" + EnvVarAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY" + EnvVarAWSVPCID = "AWS_VPC_ID" + EnvVarAWSInstanceType = "AWS_INSTANCE_TYPE" + EnvVarAWSSecurityGroupID = "AWS_SG_ID" + EnvVarAWSClusterIdentity = "AWS_CLUSTER_IDENTITY" + EnvVarAWSSubnets = "AWS_SUBNETS" // VSphere EnvVarVSphereUser = "VSPHERE_USER" diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go index fbee92d44..6555cf177 100644 --- a/test/e2e/managedcluster/managedcluster.go +++ b/test/e2e/managedcluster/managedcluster.go @@ -127,9 +127,7 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured { // since we populate the vars from standalone prior to this step. ValidateDeploymentVars([]string{ EnvVarAWSVPCID, - EnvVarAWSSubnetID, - EnvVarAWSSubnetAvailabilityZone, - EnvVarAWSSecurityGroupID, + EnvVarAWSSubnets, }) managedClusterTemplateBytes = awsHostedCPManagedClusterTemplateBytes case TemplateVSphereStandaloneCP: From 3d536720e30bda81f4e2e593762167be925c8851 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Sat, 30 Nov 2024 06:30:48 -0800 Subject: [PATCH 13/23] Set the indentation in the yaml encoding of aws subnets --- test/e2e/managedcluster/aws/aws.go | 9 +++++++-- test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index c655f54ca..fa8adb98c 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -17,6 +17,7 @@ package aws import ( + "bytes" "context" . "github.com/onsi/ginkgo/v2" @@ -65,9 +66,13 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, } } - subnetYaml, err := yaml.Marshal(subnetMaps) + buf := bytes.Buffer{} + enc := yaml.NewEncoder(&buf) + enc.SetIndent(5) + err = enc.Encode(&subnetMaps) Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, string(subnetYaml)) + subnetYaml := buf.String() + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, subnetYaml) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") diff --git a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl index feac07edf..cf67c0e17 100644 --- a/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/aws-hosted-cp.yaml.tpl @@ -12,7 +12,7 @@ spec: vpcID: ${AWS_VPC_ID} region: ${AWS_REGION} subnets: - ${AWS_SUBNETS} +${AWS_SUBNETS} instanceType: ${AWS_INSTANCE_TYPE:=t3.medium} securityGroupIDs: - ${AWS_SG_ID} From 53613cd4dd5faaf99105c3a810d9af82fc952f3e Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Sat, 30 Nov 2024 10:50:53 -0800 Subject: [PATCH 14/23] Format the yaml --- test/e2e/managedcluster/aws/aws.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index fa8adb98c..6303479ae 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -17,8 +17,10 @@ package aws import ( - "bytes" + "bufio" "context" + "fmt" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -65,14 +67,14 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, "id": subnet["resourceID"], } } - - buf := bytes.Buffer{} - enc := yaml.NewEncoder(&buf) - enc.SetIndent(5) - err = enc.Encode(&subnetMaps) + var subnetsFormatted string + encodedYaml, err := yaml.Marshal(subnetMaps) Expect(err).NotTo(HaveOccurred(), "failed to get marshall subnet maps") - subnetYaml := buf.String() - GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, subnetYaml) + scanner := bufio.NewScanner(strings.NewReader(string(encodedYaml))) + for scanner.Scan() { + subnetsFormatted += fmt.Sprintf(" %s\n", scanner.Text()) + } + GinkgoT().Setenv(managedcluster.EnvVarAWSSubnets, subnetsFormatted) securityGroupID, found, err := unstructured.NestedString( awsCluster.Object, "status", "networkStatus", "securityGroups", "node", "id") From f92fc366560fec0896a7f829f47584a2d4fe6a05 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 2 Dec 2024 10:58:07 -0800 Subject: [PATCH 15/23] Change hosted azure template to include control plane subnet --- .../templates/azurecluster.yaml | 12 +++++-- .../azure-hosted-cp/values.schema.json | 5 ++- templates/cluster/azure-hosted-cp/values.yaml | 7 ++-- test/e2e/managedcluster/azure/azure.go | 35 ++++++++++--------- .../resources/azure-hosted-cp.yaml.tpl | 7 ++-- 5 files changed, 41 insertions(+), 25 deletions(-) diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index 2bdb10501..56f246654 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -18,11 +18,17 @@ spec: subnets: - name: {{ .Values.network.nodeSubnetName }} # TODO this is just a test, if this works we need to consider changing this to be able to specify multiple subnets - role: cluster + role: node routeTable: - name: {{ .Values.network.routeTableName }} + name: {{ .Values.network.nodeRouteTableName }} securityGroup: - name: {{ .Values.network.securityGroupName }} + name: {{ .Values.network.nodeSecurityGroupName }} + - name: {{ .Values.network.cpSubnetName }} + role: control-plane + routeTable: + name: {{ .Values.network.cpRouteTableName }} + securityGroup: + name: {{ .Values.network.cpSecurityGroupName }} location: {{ .Values.location }} {{- if .Values.bastion.enabled }} {{- with .Values.bastion.bastionSpec }} diff --git a/templates/cluster/azure-hosted-cp/values.schema.json b/templates/cluster/azure-hosted-cp/values.schema.json index dd04d037b..18f42ef01 100644 --- a/templates/cluster/azure-hosted-cp/values.schema.json +++ b/templates/cluster/azure-hosted-cp/values.schema.json @@ -101,7 +101,10 @@ "vnetName", "nodeSubnetName", "routeTableName", - "securityGroupName" + "securityGroupName", + "cpSubnetName", + "cpRouteTableName", + "cpSecurityGroupName" ], "properties": { "vnetName": { diff --git a/templates/cluster/azure-hosted-cp/values.yaml b/templates/cluster/azure-hosted-cp/values.yaml index b4e1b81e6..075fbb91f 100644 --- a/templates/cluster/azure-hosted-cp/values.yaml +++ b/templates/cluster/azure-hosted-cp/values.yaml @@ -24,8 +24,11 @@ resourceGroup: "" network: vnetName: "" nodeSubnetName: "" - routeTableName: "" - securityGroupName: "" + nodeRouteTableName: "" + nodeSecurityGroupName: "" + cpSubnetName: "" + cpRouteTableName: "" + cpSecurityGroupName: "" # Azure machines parameters sshPublicKey: "" diff --git a/test/e2e/managedcluster/azure/azure.go b/test/e2e/managedcluster/azure/azure.go index ed53e6f5f..02d496f9a 100644 --- a/test/e2e/managedcluster/azure/azure.go +++ b/test/e2e/managedcluster/azure/azure.go @@ -87,26 +87,27 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) Expect(exists).To(BeTrue()) routeTableName := routeTable["name"] - if routeTableName != nil && len(fmt.Sprintf("%s", routeTableName)) > 0 { - subnetMap = sMap - break - } - } + subnetName := sMap["name"] - subnetName := subnetMap["name"] - GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) + securityGroup, found, err := unstructured.NestedMap(subnetMap, "securityGroup") + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + securityGroupName := securityGroup["name"] - securityGroup, found, err := unstructured.NestedMap(subnetMap, "securityGroup") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - securityGroupName := securityGroup["name"] - GinkgoT().Setenv("AZURE_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + role, exists, err := unstructured.NestedString(sMap, "role") + Expect(err).NotTo(HaveOccurred()) + Expect(exists).To(BeTrue()) - routeTable, found, err := unstructured.NestedMap(subnetMap, "routeTable") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - routeTableName := routeTable["name"] - GinkgoT().Setenv("AZURE_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + if role == "control-plane" { + GinkgoT().Setenv("AZURE_CP_SUBNET", fmt.Sprintf("%s", subnetName)) + GinkgoT().Setenv("AZURE_CP_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + GinkgoT().Setenv("AZURE_CP_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + } else { + GinkgoT().Setenv("AZURE_NODE_SUBNET", fmt.Sprintf("%s", subnetName)) + GinkgoT().Setenv("AZURE_NODE_SECURITY_GROUP", fmt.Sprintf("%s", securityGroupName)) + GinkgoT().Setenv("AZURE_NODE_ROUTE_TABLE", fmt.Sprintf("%s", routeTableName)) + } + } } // CreateDefaultStorageClass configures the default storage class for Azure diff --git a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl index 407096ab2..919a7c25b 100644 --- a/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl +++ b/test/e2e/managedcluster/resources/azure-hosted-cp.yaml.tpl @@ -17,8 +17,11 @@ spec: network: vnetName: "${AZURE_VM_NET_NAME}" nodeSubnetName: "${AZURE_NODE_SUBNET}" - routeTableName: "${AZURE_ROUTE_TABLE}" - securityGroupName: "${AZURE_SECURITY_GROUP}" + nodeRouteTableName: "${AZURE_NODE_ROUTE_TABLE}" + nodeSecurityGroupName: "${AZURE_NODE_SECURITY_GROUP}" + cpSubnetName: "${AZURE_CP_SUBNET}" + cpRouteTableName: "${AZURE_CP_ROUTE_TABLE}" + cpSecurityGroupName: "${AZURE_CP_SECURITY_GROUP}" tenantID: "${AZURE_TENANT_ID}" clientID: "${AZURE_CLIENT_ID}" clientSecret: "${AZURE_CLIENT_SECRET}" From 9822ec11ed3d7542d01059c5ac60196de25888d3 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 2 Dec 2024 11:08:57 -0800 Subject: [PATCH 16/23] Update schema --- templates/cluster/azure-hosted-cp/values.schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/cluster/azure-hosted-cp/values.schema.json b/templates/cluster/azure-hosted-cp/values.schema.json index 18f42ef01..75e3947fe 100644 --- a/templates/cluster/azure-hosted-cp/values.schema.json +++ b/templates/cluster/azure-hosted-cp/values.schema.json @@ -100,8 +100,8 @@ "required": [ "vnetName", "nodeSubnetName", - "routeTableName", - "securityGroupName", + "nodeRouteTableName", + "nodeSecurityGroupName", "cpSubnetName", "cpRouteTableName", "cpSecurityGroupName" From 510c6df61c96be0bb81f277c6a329a0200e58aa3 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 2 Dec 2024 12:13:38 -0800 Subject: [PATCH 17/23] Add debug logging to ccm check --- test/e2e/managedcluster/validate_deployed.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/managedcluster/validate_deployed.go b/test/e2e/managedcluster/validate_deployed.go index bae823f75..195293e5d 100644 --- a/test/e2e/managedcluster/validate_deployed.go +++ b/test/e2e/managedcluster/validate_deployed.go @@ -266,6 +266,8 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str } for _, i := range service.Status.LoadBalancer.Ingress { + fmt.Printf("HOSTNAME=%s, IP=%s", i.Hostname, i.IP) + fmt.Printf("service status=%v", service.Status) if i.Hostname != "" || i.IP != "" { return nil } From 469f79dbb1477c3c50735133e498b40ae1631868 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 2 Dec 2024 12:40:35 -0800 Subject: [PATCH 18/23] Fix up azure env var processing --- test/e2e/managedcluster/azure/azure.go | 3 +-- test/e2e/managedcluster/validate_deployed.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/test/e2e/managedcluster/azure/azure.go b/test/e2e/managedcluster/azure/azure.go index 02d496f9a..9056e793e 100644 --- a/test/e2e/managedcluster/azure/azure.go +++ b/test/e2e/managedcluster/azure/azure.go @@ -77,7 +77,6 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) resourceGroup := spec["resourceGroup"] GinkgoT().Setenv("AZURE_RESOURCE_GROUP", fmt.Sprintf("%s", resourceGroup)) - var subnetMap map[string]any for _, subnet := range subnets { sMap, ok := subnet.(map[string]any) Expect(ok).To(BeTrue()) @@ -89,7 +88,7 @@ func SetAzureEnvironmentVariables(clusterName string, kc *kubeclient.KubeClient) subnetName := sMap["name"] - securityGroup, found, err := unstructured.NestedMap(subnetMap, "securityGroup") + securityGroup, found, err := unstructured.NestedMap(sMap, "securityGroup") Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) securityGroupName := securityGroup["name"] diff --git a/test/e2e/managedcluster/validate_deployed.go b/test/e2e/managedcluster/validate_deployed.go index 195293e5d..aa5fca51d 100644 --- a/test/e2e/managedcluster/validate_deployed.go +++ b/test/e2e/managedcluster/validate_deployed.go @@ -266,8 +266,8 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str } for _, i := range service.Status.LoadBalancer.Ingress { - fmt.Printf("HOSTNAME=%s, IP=%s", i.Hostname, i.IP) - fmt.Printf("service status=%v", service.Status) + _, _ = fmt.Fprintf(GinkgoWriter, "HOSTNAME=%s, IP=%s", i.Hostname, i.IP) + _, _ = fmt.Fprintf(GinkgoWriter, "service status=%v", service.Status) if i.Hostname != "" || i.IP != "" { return nil } From 4f72ce6def2aa42cec5a04f216b9a99583c72d02 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 2 Dec 2024 16:52:12 -0800 Subject: [PATCH 19/23] attempt to remove subnet spec as a test --- .../templates/azurecluster.yaml | 27 +++++++++---------- test/e2e/provider_aws_test.go | 16 ++--------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index 56f246654..f49e45eb5 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -15,20 +15,19 @@ spec: vnet: resourceGroup: {{ .Values.resourceGroup }} name: {{ .Values.network.vnetName }} - subnets: - - name: {{ .Values.network.nodeSubnetName }} - # TODO this is just a test, if this works we need to consider changing this to be able to specify multiple subnets - role: node - routeTable: - name: {{ .Values.network.nodeRouteTableName }} - securityGroup: - name: {{ .Values.network.nodeSecurityGroupName }} - - name: {{ .Values.network.cpSubnetName }} - role: control-plane - routeTable: - name: {{ .Values.network.cpRouteTableName }} - securityGroup: - name: {{ .Values.network.cpSecurityGroupName }} +# subnets: +# - name: {{ .Values.network.nodeSubnetName }} +# role: node +# routeTable: +# name: {{ .Values.network.nodeRouteTableName }} +# securityGroup: +# name: {{ .Values.network.nodeSecurityGroupName }} +# - name: {{ .Values.network.cpSubnetName }} +# role: control-plane +# routeTable: +# name: {{ .Values.network.cpRouteTableName }} +# securityGroup: +# name: {{ .Values.network.cpSecurityGroupName }} location: {{ .Values.location }} {{- if .Values.bastion.enabled }} {{- with .Values.bastion.bastionSpec }} diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 52586eba3..6614698b4 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -23,7 +23,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/utils/env" internalutils "github.com/Mirantis/hmc/internal/utils" "github.com/Mirantis/hmc/test/e2e/kubeclient" @@ -107,9 +106,8 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order kubeCfgPath, kubecfgDeleteFunc = kc.WriteKubeconfig(context.Background(), clusterName) GinkgoT().Setenv("KUBECONFIG", kubeCfgPath) - cmd := exec.Command("make", fmt.Sprintf("VERSION=%s", env.GetString("VERSION", "")), "test-apply") - output, err := utils.Run(cmd) - _, _ = fmt.Fprint(GinkgoWriter, string(output)) + cmd := exec.Command("make", "test-apply") + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred()) Expect(os.Unsetenv("KUBECONFIG")).To(Succeed()) @@ -126,16 +124,6 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - Eventually(func() error { - By("Ensure cluster templates valid") - err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) - if err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) - return err - } - return nil - }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) - // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS) From d57597b7f51d1b6a61b2ac65a26a41168f7b7ed6 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 4 Dec 2024 09:40:48 -0800 Subject: [PATCH 20/23] Add additional networking info --- test/e2e/managedcluster/aws/aws.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index 6303479ae..c79eecba7 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -65,6 +65,9 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, "isPublic": subnet["isPublic"], "availabilityZone": subnet["availabilityZone"], "id": subnet["resourceID"], + "natGatewayId": subnet["natGatewayId"], + "routeTableId": subnet["routeTableId"], + "zoneType": subnet["availability-zone"], } } var subnetsFormatted string From 6dc051f17f44621b98e58bd689089a347f007ca5 Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 4 Dec 2024 12:12:29 -0800 Subject: [PATCH 21/23] Do not set gateway id unless it exists and fix up zone --- test/e2e/managedcluster/aws/aws.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/e2e/managedcluster/aws/aws.go b/test/e2e/managedcluster/aws/aws.go index c79eecba7..3b051aa6b 100644 --- a/test/e2e/managedcluster/aws/aws.go +++ b/test/e2e/managedcluster/aws/aws.go @@ -65,9 +65,12 @@ func PopulateHostedTemplateVars(ctx context.Context, kc *kubeclient.KubeClient, "isPublic": subnet["isPublic"], "availabilityZone": subnet["availabilityZone"], "id": subnet["resourceID"], - "natGatewayId": subnet["natGatewayId"], "routeTableId": subnet["routeTableId"], - "zoneType": subnet["availability-zone"], + "zoneType": "availability-zone", + } + + if natGatewayID, exists := subnet["natGatewayId"]; exists && natGatewayID != "" { + subnetMaps[i]["natGatewayId"] = natGatewayID } } var subnetsFormatted string From 70f2321761a241a6ec39d0bd479886be57340bde Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Wed, 4 Dec 2024 14:14:03 -0800 Subject: [PATCH 22/23] Revert azure networking change --- .../templates/azurecluster.yaml | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index f49e45eb5..8b94133d8 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -15,19 +15,19 @@ spec: vnet: resourceGroup: {{ .Values.resourceGroup }} name: {{ .Values.network.vnetName }} -# subnets: -# - name: {{ .Values.network.nodeSubnetName }} -# role: node -# routeTable: -# name: {{ .Values.network.nodeRouteTableName }} -# securityGroup: -# name: {{ .Values.network.nodeSecurityGroupName }} -# - name: {{ .Values.network.cpSubnetName }} -# role: control-plane -# routeTable: -# name: {{ .Values.network.cpRouteTableName }} -# securityGroup: -# name: {{ .Values.network.cpSecurityGroupName }} + subnets: + - name: {{ .Values.network.nodeSubnetName }} + role: node + routeTable: + name: {{ .Values.network.nodeRouteTableName }} + securityGroup: + name: {{ .Values.network.nodeSecurityGroupName }} + - name: {{ .Values.network.cpSubnetName }} + role: control-plane + routeTable: + name: {{ .Values.network.cpRouteTableName }} + securityGroup: + name: {{ .Values.network.cpSecurityGroupName }} location: {{ .Values.location }} {{- if .Values.bastion.enabled }} {{- with .Values.bastion.bastionSpec }} From f83cff8d08bbdc4d6fa7dfd927070617fe74927a Mon Sep 17 00:00:00 2001 From: Kyle Wuolle Date: Mon, 9 Dec 2024 13:42:44 -0800 Subject: [PATCH 23/23] Add template check --- test/e2e/e2e_suite_test.go | 2 +- test/e2e/provider_aws_test.go | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 45290570d..bc6ec680d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -74,7 +74,7 @@ var _ = BeforeSuite(func() { Eventually(func() error { err = managedcluster.ValidateClusterTemplates(context.Background(), kc) if err != nil { - _, _ = fmt.Fprintf(GinkgoWriter, "cluster tempolate validation failed: %v\n", err) + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) return err } return nil diff --git a/test/e2e/provider_aws_test.go b/test/e2e/provider_aws_test.go index 6614698b4..ee0abe619 100644 --- a/test/e2e/provider_aws_test.go +++ b/test/e2e/provider_aws_test.go @@ -124,6 +124,14 @@ var _ = Describe("AWS Templates", Label("provider:cloud", "provider:aws"), Order return nil }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) + Eventually(func() error { + err = managedcluster.ValidateClusterTemplates(context.Background(), standaloneClient) + if err != nil { + _, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err) + return err + } + return nil + }).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed()) // Ensure AWS credentials are set in the standalone cluster. clusteridentity.New(standaloneClient, managedcluster.ProviderAWS)