E2E tests with EKS

Mirantis · Nov 21, 2024 · a623d6b · a623d6b
1 parent 2b19b51
commit a623d6b
Show file tree

Hide file tree

Showing 16 changed files with 225 additions and 70 deletions.
diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -1,6 +1,6 @@
 name: CI
 on:
-  pull_request_target:
+  pull_request:
     types:
       - labeled
       - opened
@@ -23,29 +23,14 @@ env:
 jobs:
   build:
     concurrency:
-      group: build-${{ github.head_ref || github.run_id }}
+      group: ${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     name: Build and Unit Test
     runs-on: ubuntu-latest
-    outputs:
-      version: ${{ steps.vars.outputs.version }}
-      clustername: ${{ steps.vars.outputs.clustername }}
-      pr: ${{ steps.pr.outputs.result }}
     steps:
-      - name: Get PR ref
-        uses: actions/github-script@v7
-        id: pr
-        with:
-          script: |
-            const { data: pullRequest } = await github.rest.pulls.get({
-              ...context.repo,
-              pull_number: context.payload.pull_request.number,
-            });
-            return pullRequest
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
-          ref: ${{fromJSON(steps.pr.outputs.result).merge_commit_sha}}
           fetch-depth: 0
       - name: Setup Go
         uses: actions/setup-go@v5
@@ -59,6 +44,31 @@ jobs:
       - name: Unit tests
         run: |
           make test
+      - name: Build HMC controller image
+        run: |
+          make docker-build
+
+  push:
+    concurrency:
+      group: push-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    name: E2E Push Images and Charts to GHCR
+    runs-on: ubuntu-latest
+    needs: build
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
+    outputs:
+      version: ${{ steps.vars.outputs.version }}
+      clustername: ${{ steps.vars.outputs.clustername }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+          cache: false
       - name: Set up Buildx
         uses: docker/setup-buildx-action@v3
       - name: Login to GHCR
@@ -73,7 +83,8 @@ jobs:
           GIT_VERSION=$(git describe --tags --always)
           echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT
           echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
-      - name: Build and push HMC controller image
+      - name: Push HMC Controller Image to GHCR
+        if:
         uses: docker/build-push-action@v6
         with:
           build-args: |
@@ -85,36 +96,35 @@ jobs:
           push: true
           cache-from: type=gha
           cache-to: type=gha,mode=max
-      - name: Prepare and push HMC template charts
+      - name: Prepare and push HMC template charts to GHCR
         run: |
           make hmc-chart-release
           make helm-push
 
   controller-e2etest:
     name: E2E Controller
     runs-on: ubuntu-latest
-    needs: build
+    if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
+    needs: push
     concurrency:
       group: controller-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup kubectl
         uses: azure/setup-kubectl@v4
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'controller'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -129,14 +139,13 @@ jobs:
     name: E2E Cloud Providers
     runs-on: ubuntu-latest
     if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
-    needs: build
+    needs: push
     concurrency:
       group: cloud-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     env:
       AWS_REGION: us-west-2
       AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
@@ -151,7 +160,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -162,9 +170,9 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:cloud'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -179,14 +187,13 @@ jobs:
     name: E2E On-Prem Providers
     runs-on: self-hosted
     if: ${{ contains( github.event.pull_request.labels.*.name, 'test e2e') }}
-    needs: build
+    needs: push
     concurrency:
       group: onprem-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     env:
       VSPHERE_USER: ${{ secrets.CI_VSPHERE_USER }}
       VSPHERE_PASSWORD: ${{ secrets.CI_VSPHERE_PASSWORD }}
@@ -205,7 +212,6 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -215,9 +221,9 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:onprem'
-          MANAGED_CLUSTER_NAME: ${{ needs.build.outputs.clustername }}
-          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
-          VERSION: ${{ needs.build.outputs.version }}
+          MANAGED_CLUSTER_NAME: ${{ needs.push.outputs.clustername }}
+          IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.push.outputs.version }}'
+          VERSION: ${{ needs.push.outputs.version }}
         run: |
           make test-e2e
       - name: Archive test results
@@ -229,23 +235,21 @@ jobs:
             test/e2e/*.log
 
   cleanup:
-    name: Cleanup
+    name: E2E Cleanup
     needs:
-      - build
+      - push
       - provider-cloud-e2etest
     runs-on: ubuntu-latest
-    if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }}
+    if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.push.result, 'success') }}
     timeout-minutes: 15
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
-      version: ${{ needs.build.outputs.version }}
-      pr: ${{ needs.build.outputs.pr }}
+      clustername: ${{ needs.push.outputs.clustername }}
+      version: ${{ needs.push.outputs.version }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
-          ref: ${{fromJSON(needs.build.outputs.pr).merge_commit_sha}}
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
@@ -260,7 +264,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
-          CLUSTER_NAME: '${{ needs.build.outputs.clustername }}'
+          CLUSTER_NAME: '${{ needs.push.outputs.clustername }}'
         run: |
           make dev-aws-nuke
-          make dev-azure-nuke
+          make dev-azure-nuke
diff --git a/config/dev/eks-managedcluster.yaml b/config/dev/eks-managedcluster.yaml
@@ -4,7 +4,7 @@ metadata:
   name: eks-dev
   namespace: ${NAMESPACE}
 spec:
-  template: aws-eks-0-0-2
+  template: aws-eks-0-0-3
   credential: "aws-cluster-identity-cred"
   config:
     region: ${AWS_REGION}

diff --git a/templates/cluster/aws-eks/Chart.yaml b/templates/cluster/aws-eks/Chart.yaml
@@ -6,7 +6,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.0.2
+version: 0.0.3
 annotations:
   cluster.x-k8s.io/provider: infrastructure-aws
   cluster.x-k8s.io/infrastructure-aws: v1beta2
diff --git a/...plates/files/templates/aws-eks-0-0-2.yaml → ...plates/files/templates/aws-eks-0-0-3.yaml b/...plates/files/templates/aws-eks-0-0-2.yaml → ...plates/files/templates/aws-eks-0-0-3.yaml
@@ -1,10 +1,10 @@
 apiVersion: hmc.mirantis.com/v1alpha1
 kind: ClusterTemplate
 metadata:
-  name: aws-eks-0-0-2
+  name: aws-eks-0-0-3
   annotations:
     helm.sh/resource-policy: keep
 spec:
   helm:
     chartName: aws-eks
-    chartVersion: 0.0.2
+    chartVersion: 0.0.3
diff --git a/templates/provider/hmc/templates/rbac/controller/roles.yaml b/templates/provider/hmc/templates/rbac/controller/roles.yaml
@@ -191,6 +191,11 @@ rules:
   - clusterprofiles
   - clustersummaries
   verbs: {{ include "rbac.editorVerbs" . | nindent 4 }}
+- apiGroups:
+  - controlplane.cluster.x-k8s.io
+  resources:
+  - awsmanagedcontrolplanes
+  verbs: {{ include "rbac.viewerVerbs" . | nindent 4 }}
 - apiGroups:
   - hmc.mirantis.com
   resources:

diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -217,8 +217,10 @@ func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, provider
 			"describe", "cluster", clusterName, "--namespace", internalutils.DefaultSystemNamespace, "--show-conditions=all")
 		output, err := utils.Run(cmd)
 		if err != nil {
-			utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err))
-			return
+			if !strings.Contains(err.Error(), "unable to verify clusterctl version") {
+				utils.WarnError(fmt.Errorf("failed to get clusterctl log: %w", err))
+				return
+			}
 		}
 		err = os.WriteFile(filepath.Join("test/e2e", host+"-"+"clusterctl.log"), output, 0o644)
 		if err != nil {

diff --git a/test/e2e/kubeclient/kubeclient.go b/test/e2e/kubeclient/kubeclient.go
@@ -279,3 +279,15 @@ func (kc *KubeClient) ListK0sControlPlanes(
 		Resource: "k0scontrolplanes",
 	}, clusterName)
 }
+
+func (kc *KubeClient) ListAWSManagedControlPlanes(
+	ctx context.Context, clusterName string,
+) ([]unstructured.Unstructured, error) {
+	GinkgoHelper()
+
+	return kc.listResource(ctx, schema.GroupVersionResource{
+		Group:    "controlplane.cluster.x-k8s.io",
+		Version:  "v1beta2",
+		Resource: "awsmanagedcontrolplanes",
+	}, clusterName)
+}
diff --git a/test/e2e/managedcluster/managedcluster.go b/test/e2e/managedcluster/managedcluster.go
@@ -50,6 +50,7 @@ const (
 	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
 	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
 	TemplateVSphereHostedCP     Template = "vsphere-hosted-cp"
+	TemplateEKSCP               Template = "aws-eks-cp"
 )
 
 //go:embed resources/aws-standalone-cp.yaml.tpl
@@ -70,6 +71,9 @@ var vsphereStandaloneCPManagedClusterTemplateBytes []byte
 //go:embed resources/vsphere-hosted-cp.yaml.tpl
 var vsphereHostedCPManagedClusterTemplateBytes []byte
 
+//go:embed resources/aws-eks-cp.yaml.tpl
+var eksCPManagedClusterTemplateBytes []byte
+
 func FilterAllProviders() []string {
 	return []string{
 		utils.HMCControllerLabel,
@@ -134,6 +138,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		managedClusterTemplateBytes = azureHostedCPManagedClusterTemplateBytes
 	case TemplateAzureStandaloneCP:
 		managedClusterTemplateBytes = azureStandaloneCPManagedClusterTemplateBytes
+	case TemplateEKSCP:
+		managedClusterTemplateBytes = eksCPManagedClusterTemplateBytes
 	default:
 		Fail(fmt.Sprintf("Unsupported template: %s", templateName))
 	}

diff --git a/test/e2e/managedcluster/providervalidator.go b/test/e2e/managedcluster/providervalidator.go
@@ -65,14 +65,22 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
+		case TemplateEKSCP:
+			resourcesToValidate["control-planes"] = validateAWSManagedControlPlanes
+			delete(resourcesToValidate, "csi-driver")
 		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
 		}
 	} else {
+		validateCPDeletedFunc := validateK0sControlPlanesDeleted
+		if template == TemplateEKSCP {
+			validateCPDeletedFunc = validateAWSManagedControlPlanesDeleted
+		}
+
 		resourcesToValidate = map[string]resourceValidationFunc{
 			"clusters":           validateClusterDeleted,
 			"machinedeployments": validateMachineDeploymentsDeleted,
-			"control-planes":     validateK0sControlPlanesDeleted,
+			"control-planes":     validateCPDeletedFunc,
 		}
 		resourceOrder = []string{"clusters", "machinedeployments", "control-planes"}
 	}

diff --git a/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl b/test/e2e/managedcluster/resources/aws-eks-cp.yaml.tpl
@@ -0,0 +1,16 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ManagedCluster
+metadata:
+  name: ${MANAGED_CLUSTER_NAME}-eks
+spec:
+  template: aws-eks-0-0-3
+  credential: ${AWS_CLUSTER_IDENTITY}-cred
+  config:
+    region: ${AWS_REGION}
+    workersNumber: ${WORKERS_NUMBER:=1}
+    clusterIdentity:
+      name:  ${AWS_CLUSTER_IDENTITY}-cred
+      namespace: ${NAMESPACE}
+    publicIP: ${AWS_PUBLIC_IP:=true}
+    worker:
+      instanceType: ${AWS_INSTANCE_TYPE:=t3.small}