feat: add e2e tests for GCP with workload identity (kedacore#3916)

Fixes kedacore#3897
Fixes kedacore#3898
Fixes kedacore#3899

Makefile

@@ -36,6 +36,9 @@ GIT_COMMIT  ?= $(shell git rev-list -1 HEAD)
 DATE        = $(shell date -u +"%Y.%m.%d.%H.%M.%S")
 
 TEST_CLUSTER_NAME ?= keda-nightly-run-3
+NON_ROOT_USER_ID ?= 1000
+
+GCP_WI_PROVIDER ?= projects/${TF_GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${TEST_CLUSTER_NAME}/providers/${TEST_CLUSTER_NAME}
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -236,6 +239,11 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 		cd config/service_account && \
 		$(KUSTOMIZE) edit add annotation --force eks.amazonaws.com/role-arn:arn:aws:iam::${TF_AWS_ACCOUNT_ID}:role/${TEST_CLUSTER_NAME}-role; \
 	fi
+	if [ "$(GCP_RUN_IDENTITY_TESTS)" = true ]; then \
+		cd config/service_account && \
+		$(KUSTOMIZE) edit add annotation --force cloud.google.com/workload-identity-provider:${GCP_WI_PROVIDER} cloud.google.com/service-account-email:${TF_GCP_SA_EMAIL} cloud.google.com/gcloud-run-as-user:${NON_ROOT_USER_ID}; \
+	fi
+
 	# Need this workaround to mitigate a problem with inserting labels into selectors,
 	# until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009
 	@sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml

pkg/scalers/gcp_stackdriver_client.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
+	"os"
 	"strings"
 	"time"
 
@@ -53,10 +54,16 @@ func NewStackDriverClientPodIdentity(ctx context.Context) (*StackDriverClient, e
 		return nil, err
 	}
 	c := metadata.NewClient(&http.Client{})
-	project, err := c.ProjectID()
-	if err != nil {
-		return nil, err
+
+	// Running workload identity outside GKE, we can't use the metadata api and we need to use the env that it's provided from the hook
+	project, found := os.LookupEnv("CLOUDSDK_CORE_PROJECT")
+	if !found {
+		project, err = c.ProjectID()
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	return &StackDriverClient{
 		metricsClient: client,
 		projectID:     project,

tests/.env

@@ -27,7 +27,7 @@ TF_AZURE_SUBSCRIPTION=
 DATADOG_APP_KEY=
 DATADOG_API_KEY=
 DATADOG_SITE=
-GCP_SP_KEY=
+TF_GCP_SA_CREDENTIALS=
 OPENSTACK_AUTH_URL=
 OPENSTACK_PASSWORD=
 OPENSTACK_PROJECT_ID=

tests/helper/helper.go

@@ -35,6 +35,8 @@ import (
 const (
 	AzureWorkloadIdentityNamespace = "azure-workload-identity-system"
 	AwsIdentityNamespace           = "aws-identity-system"
+	GcpIdentityNamespace           = "gcp-identity-system"
+	CertManagerNamespace           = "cert-manager"
 	KEDANamespace                  = "keda"
 	KEDAOperator                   = "keda-operator"
 	KEDAMetricsAPIServer           = "keda-metrics-apiserver"
@@ -54,6 +56,8 @@ var (
 	AzureADTenantID               = os.Getenv("TF_AZURE_SP_TENANT")
 	AzureRunWorkloadIdentityTests = os.Getenv("AZURE_RUN_WORKLOAD_IDENTITY_TESTS")
 	AwsIdentityTests              = os.Getenv("AWS_RUN_IDENTITY_TESTS")
+	GcpIdentityTests              = os.Getenv("GCP_RUN_IDENTITY_TESTS")
+	InstallCertManager            = AwsIdentityTests == StringTrue || GcpIdentityTests == StringTrue
 )
 
 var (

tests/scalers/gcp_pubsub/gcp_pubsub_test.go → tests/scalers/gcp/gcp_pubsub/gcp_pubsub_test.go

@@ -29,7 +29,7 @@ const (
 )
 
 var (
-	gcpKey              = os.Getenv("GCP_SP_KEY")
+	gcpKey              = os.Getenv("TF_GCP_SA_CREDENTIALS")
 	creds               = make(map[string]interface{})
 	errGcpKey           = json.Unmarshal([]byte(gcpKey), &creds)
 	testNamespace       = fmt.Sprintf("%s-ns", testName)
@@ -168,7 +168,7 @@ spec:
 func TestScaler(t *testing.T) {
 	// setup
 	t.Log("--- setting up ---")
-	require.NotEmpty(t, gcpKey, "GCP_KEY env variable is required for GCP storage test")
+	require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test")
 	assert.NoErrorf(t, errGcpKey, "Failed to load credentials from gcpKey - %s", errGcpKey)
 
 	// Create kubernetes resources