feat: Integration tests fleetscope (#61)

Co-authored-by: Samir Ribeiro <[email protected]> Co-authored-by: Samir Ribeiro <[email protected]> Co-authored-by: Andrew Peabody <[email protected]>
GoogleCloudPlatform · Apr 4, 2024 · 454618c · 454618c
1 parent 0976081
commit 454618c
Show file tree

Hide file tree

Showing 3 changed files with 107 additions and 2 deletions.
diff --git a/test/integration/fleetscope/fleetscope_test.go b/test/integration/fleetscope/fleetscope_test.go
@@ -16,10 +16,13 @@ package fleetscope
 
 import (
 	"fmt"
+	"regexp"
 	"testing"
 	"time"
 
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/stretchr/testify/assert"
 	"github.com/terraform-google-modules/enterprise-application/test/integration/testutils"
 )
 
@@ -53,13 +56,115 @@ func TestFleetscope(t *testing.T) {
 				"network_project_id":     multitenant.GetStringOutput("network_project_id"),
 				"cluster_membership_ids": multitenant.GetStringOutputList("cluster_membership_ids"),
 			}
+
 			fleetscope := tft.NewTFBlueprintTest(t,
 				tft.WithTFDir(fmt.Sprintf("../../../4-fleetscope/envs/%s", envName)),
 				tft.WithVars(vars),
 				tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
 				tft.WithBackendConfig(backendConfig),
 			)
 
+			fleetscope.DefineVerify(func(assert *assert.Assertions) {
+				fleetscope.DefaultVerify(assert)
+
+				// Multitenant Outputs
+				fleetProjectID := multitenant.GetStringOutput("fleet_project_id")
+				clusterRegions := multitenant.GetStringOutputList("cluster_regions")
+				clusterIds := multitenant.GetStringOutputList("clusters_ids")
+				clusterProjectID := multitenant.GetStringOutput("cluster_project_id")
+
+				// Service Account
+				rootReconcilerRoles := []string{"roles/source.reader"}
+				rootReconcilerSa := fmt.Sprintf("root-reconciler@%s.iam.gserviceaccount.com", fleetProjectID)
+				iamReconcilerFilter := fmt.Sprintf("bindings.members:'serviceAccount:%s'", rootReconcilerSa)
+				iamReconcilerCommonArgs := gcloud.WithCommonArgs([]string{"--flatten", "bindings", "--filter", iamReconcilerFilter, "--format", "json"})
+				projectPolicyOp := gcloud.Run(t, fmt.Sprintf("projects get-iam-policy %s", fleetProjectID), iamReconcilerCommonArgs).Array()
+				saReconcilerListRoles := testutils.GetResultFieldStrSlice(projectPolicyOp, "bindings.role")
+				assert.Subset(saReconcilerListRoles, rootReconcilerRoles, fmt.Sprintf("service account %s should have \"roles/source.reader\" project level role", rootReconcilerSa))
+
+				svcRoles := []string{"roles/iam.workloadIdentityUser"}
+				svcSa := fmt.Sprintf("%s.svc.id.goog[config-management-system/root-reconciler]", fleetProjectID)
+				iamSvcFilter := fmt.Sprintf("bindings.members:serviceAccount:'%s'", svcSa)
+				iamSvcCommonArgs := gcloud.WithCommonArgs([]string{"--flatten", "bindings", "--filter", iamSvcFilter, "--format", "json"})
+				svcPolicyOp := gcloud.Run(t, fmt.Sprintf("iam service-accounts get-iam-policy %s --project %s", rootReconcilerSa, fleetProjectID), iamSvcCommonArgs).Array()
+				saSvcListRoles := testutils.GetResultFieldStrSlice(svcPolicyOp, "bindings.role")
+				assert.Subset(saSvcListRoles, svcRoles, fmt.Sprintf("service account %s should have \"roles/iam.workloadIdentityUser\" project level role", svcSa))
+
+				// GKE Feature
+				for _, feature := range []string{
+					"configmanagement",
+					"servicemesh",
+					"multiclusteringress",
+					"multiclusterservicediscovery",
+				} {
+					gkeFeatureOp := gcloud.Runf(t, "container hub features describe %s --project %s", feature, fleetProjectID)
+					assert.Equal("ACTIVE", gkeFeatureOp.Get("resourceState.state").String(), fmt.Sprintf("Hub Feature %s should have resource state equal to ACTIVE", feature))
+
+					switch feature {
+					case "servicemesh":
+						// Service Mesh Management
+						{
+							assert.Equal("MANAGEMENT_AUTOMATIC", gkeFeatureOp.Get("fleetDefaultMemberConfig.mesh.management").String(), fmt.Sprintf("Hub Feature %s should have mesh menagement equal to MANAGEMENT_AUTOMATIC", feature))
+						}
+					case "multiclusteringress":
+						// Multicluster Ingress Membership
+						{
+							membershipNames := []string{}
+							for _, region := range clusterRegions {
+								membershipName := fmt.Sprintf("projects/%[1]s/locations/%[2]s/memberships/cluster-%[2]s-%[3]s", fleetProjectID, region, envName)
+								membershipNames = append(membershipNames, membershipName)
+							}
+							assert.Contains(membershipNames, gkeFeatureOp.Get("spec.multiclusteringress.configMembership").String(), fmt.Sprintf("Hub Feature %s should have Config Membership in one region", feature))
+						}
+					case "configmanagement":
+						// GKE Feature Membership
+						{
+							for _, region := range clusterRegions {
+								fleetProjectNumber := gcloud.Runf(t, "projects describe %s", fleetProjectID).Get("projectNumber").String()
+								membershipName := fmt.Sprintf("projects/%[1]s/locations/%[2]s/memberships/cluster-%[2]s-%[3]s", fleetProjectNumber, region, envName)
+								configmanagementPath := fmt.Sprintf("membershipSpecs.%s.configmanagement", membershipName)
+
+								assert.Equal("gcpserviceaccount", gkeFeatureOp.Get(configmanagementPath+".configSync.git.secretType").String(), fmt.Sprintf("Hub Feature %s should have git secret type equal to gcpserviceaccount", membershipName))
+								assert.Equal("unstructured", gkeFeatureOp.Get(configmanagementPath+".configSync.sourceFormat").String(), fmt.Sprintf("Hub Feature %s should have source format equal to unstructured", membershipName))
+								assert.Equal("1.17.2", gkeFeatureOp.Get(configmanagementPath+".version").String(), fmt.Sprintf("Hub Feature %s should have source format equal to unstructured", membershipName))
+								assert.Equal(rootReconcilerSa, gkeFeatureOp.Get(configmanagementPath+".configSync.git.gcpServiceAccountEmail").String(), fmt.Sprintf("Hub Feature %s should have git service account type equal to %s", membershipName, rootReconcilerSa))
+								assert.True(gkeFeatureOp.Get(configmanagementPath+".policyController.enabled").Bool(), fmt.Sprintf("Hub Feature %s  policy controler should be enabled", membershipName))
+								assert.True(gkeFeatureOp.Get(configmanagementPath+".policyController.referentialRulesEnabled").Bool(), fmt.Sprintf("Hub Feature %s  referencial rule should be enabled", membershipName))
+								assert.True(gkeFeatureOp.Get(configmanagementPath+".policyController.templateLibraryInstalled").Bool(), fmt.Sprintf("Hub Feature %s  template library should be installed", membershipName))
+							}
+						}
+					}
+				}
+
+				// GKE Membership binding
+				for _, id := range clusterIds {
+					// Cluster location
+					location := regexp.MustCompile(`\/locations\/([^\/]*)\/`).FindStringSubmatch(id)[1]
+					// Cluster and Membership details
+					clusterOp := gcloud.Runf(t, "container clusters describe %s --location %s --project %s", id, location, clusterProjectID)
+					membershipOp := gcloud.Runf(t, "container fleet memberships describe %s --location %s --project %s", clusterOp.Get("name").String(), location, fleetProjectID)
+					assert.Equal(fmt.Sprintf("%s.svc.id.goog", fleetProjectID), membershipOp.Get("authority.workloadIdentityPool").String(), fmt.Sprintf("Membership %s workloadIdentityPool should be %s.svc.id.goog", id, fleetProjectID))
+				}
+
+				// GKE Scopes and Namespaces
+				for _, namespaces := range func() []string {
+					if envName == "development" {
+						return []string{"frontend", "accounts", "transactions"}
+					}
+					return []string{"frontend"}
+				}() {
+					gkeScopes := fmt.Sprintf("projects/%s/locations/global/scopes/%s-%s", fleetProjectID, namespaces, envName)
+					opGKEScopes := gcloud.Runf(t, "container fleet scopes describe projects/%[1]s/locations/global/scopes/%[2]s-%[3]s --project=%[1]s", fleetProjectID, namespaces, envName)
+					gkeNamespaces := fmt.Sprintf("projects/%[1]s/locations/global/scopes/%[2]s-%[3]s/namespaces/%[2]s-%[3]s", fleetProjectID, namespaces, envName)
+					opNamespaces := gcloud.Runf(t, "container hub scopes namespaces describe projects/%[1]s/locations/global/scopes/%[2]s-%[3]s/namespaces/%[2]s-%[3]s --project=%[1]s", fleetProjectID, namespaces, envName)
+					assert.Equal(gkeNamespaces, opNamespaces.Get("name").String(), fmt.Sprintf("The GKE Namespace should be %s", gkeNamespaces))
+					assert.True(opNamespaces.Exists(), "Namespace %s should exist", gkeNamespaces)
+					assert.Equal(gkeScopes, opGKEScopes.Get("name").String(), fmt.Sprintf("The GKE Namespace should be %s", gkeScopes))
+					assert.True(opGKEScopes.Exists(), "Namespace %s should exist", gkeScopes)
+				}
+
+			})
+
 			fleetscope.Test()
 		})
 	}

diff --git a/test/integration/testutils/transforms.go b/test/integration/testutils/transforms.go
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2024 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

diff --git a/test/integration/testutils/utils.go b/test/integration/testutils/utils.go
@@ -1,4 +1,4 @@
-// Copyright 2022 Google LLC
+// Copyright 2024 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.