From 8f9b1d1e5f1bde9d29648aad7101adf8c056e393 Mon Sep 17 00:00:00 2001 From: Jorge Turrado Ferrero Date: Tue, 6 Dec 2022 02:20:17 +0100 Subject: [PATCH] feat: add e2e tests for GCP with workload identity (#3916) Fixes https://github.com/kedacore/keda/issues/3897 Fixes https://github.com/kedacore/keda/issues/3898 Fixes https://github.com/kedacore/keda/issues/3899 --- Makefile | 8 + pkg/scalers/gcp_stackdriver_client.go | 13 +- tests/.env | 2 +- tests/helper/helper.go | 4 + .../{ => gcp}/gcp_pubsub/gcp_pubsub_test.go | 4 +- .../gcp_pubsub_workload_identity_test.go | 308 +++++++++++++++++ .../gcp_stackdriver/gcp_stackdriver_test.go | 4 +- .../gcp_stackdriver_workload_identity_test.go | 315 ++++++++++++++++++ .../{ => gcp}/gcp_storage/gcp_storage_test.go | 4 +- .../gcp_storage_workload_identity_test.go | 276 +++++++++++++++ tests/utils/cleanup_test.go | 27 +- tests/utils/setup_test.go | 57 +++- 12 files changed, 999 insertions(+), 23 deletions(-) rename tests/scalers/{ => gcp}/gcp_pubsub/gcp_pubsub_test.go (98%) create mode 100644 tests/scalers/gcp/gcp_pubsub_workload_identity/gcp_pubsub_workload_identity_test.go rename tests/scalers/{ => gcp}/gcp_stackdriver/gcp_stackdriver_test.go (98%) create mode 100644 tests/scalers/gcp/gcp_stackdriver_workload_identity/gcp_stackdriver_workload_identity_test.go rename tests/scalers/{ => gcp}/gcp_storage/gcp_storage_test.go (97%) create mode 100644 tests/scalers/gcp/gcp_storage_workload_identity/gcp_storage_workload_identity_test.go diff --git a/Makefile b/Makefile index f39e33c70c9..873fc40a215 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,9 @@ GIT_COMMIT ?= $(shell git rev-list -1 HEAD) DATE = $(shell date -u +"%Y.%m.%d.%H.%M.%S") TEST_CLUSTER_NAME ?= keda-nightly-run-3 +NON_ROOT_USER_ID ?= 1000 + +GCP_WI_PROVIDER ?= projects/${TF_GCP_PROJECT_NUMBER}/locations/global/workloadIdentityPools/${TEST_CLUSTER_NAME}/providers/${TEST_CLUSTER_NAME} # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -236,6 +239,11 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in cd config/service_account && \ $(KUSTOMIZE) edit add annotation --force eks.amazonaws.com/role-arn:arn:aws:iam::${TF_AWS_ACCOUNT_ID}:role/${TEST_CLUSTER_NAME}-role; \ fi + if [ "$(GCP_RUN_IDENTITY_TESTS)" = true ]; then \ + cd config/service_account && \ + $(KUSTOMIZE) edit add annotation --force cloud.google.com/workload-identity-provider:${GCP_WI_PROVIDER} cloud.google.com/service-account-email:${TF_GCP_SA_EMAIL} cloud.google.com/gcloud-run-as-user:${NON_ROOT_USER_ID}; \ + fi + # Need this workaround to mitigate a problem with inserting labels into selectors, # until this issue is solved: https://github.com/kubernetes-sigs/kustomize/issues/1009 @sed -i".out" -e 's@version:[ ].*@version: $(VERSION)@g' config/default/kustomize-config/metadataLabelTransformer.yaml diff --git a/pkg/scalers/gcp_stackdriver_client.go b/pkg/scalers/gcp_stackdriver_client.go index 6f96c36cc3a..39de56ca7dc 100644 --- a/pkg/scalers/gcp_stackdriver_client.go +++ b/pkg/scalers/gcp_stackdriver_client.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "os" "strings" "time" @@ -53,10 +54,16 @@ func NewStackDriverClientPodIdentity(ctx context.Context) (*StackDriverClient, e return nil, err } c := metadata.NewClient(&http.Client{}) - project, err := c.ProjectID() - if err != nil { - return nil, err + + // Running workload identity outside GKE, we can't use the metadata api and we need to use the env that it's provided from the hook + project, found := os.LookupEnv("CLOUDSDK_CORE_PROJECT") + if !found { + project, err = c.ProjectID() + if err != nil { + return nil, err + } } + return &StackDriverClient{ metricsClient: client, projectID: project, diff --git a/tests/.env b/tests/.env index 2532b909413..f24104557ed 100644 --- a/tests/.env +++ b/tests/.env @@ -27,7 +27,7 @@ TF_AZURE_SUBSCRIPTION= DATADOG_APP_KEY= DATADOG_API_KEY= DATADOG_SITE= -GCP_SP_KEY= +TF_GCP_SA_CREDENTIALS= OPENSTACK_AUTH_URL= OPENSTACK_PASSWORD= OPENSTACK_PROJECT_ID= diff --git a/tests/helper/helper.go b/tests/helper/helper.go index e021829d3e5..6b2079fbd23 100644 --- a/tests/helper/helper.go +++ b/tests/helper/helper.go @@ -35,6 +35,8 @@ import ( const ( AzureWorkloadIdentityNamespace = "azure-workload-identity-system" AwsIdentityNamespace = "aws-identity-system" + GcpIdentityNamespace = "gcp-identity-system" + CertManagerNamespace = "cert-manager" KEDANamespace = "keda" KEDAOperator = "keda-operator" KEDAMetricsAPIServer = "keda-metrics-apiserver" @@ -54,6 +56,8 @@ var ( AzureADTenantID = os.Getenv("TF_AZURE_SP_TENANT") AzureRunWorkloadIdentityTests = os.Getenv("AZURE_RUN_WORKLOAD_IDENTITY_TESTS") AwsIdentityTests = os.Getenv("AWS_RUN_IDENTITY_TESTS") + GcpIdentityTests = os.Getenv("GCP_RUN_IDENTITY_TESTS") + InstallCertManager = AwsIdentityTests == StringTrue || GcpIdentityTests == StringTrue ) var ( diff --git a/tests/scalers/gcp_pubsub/gcp_pubsub_test.go b/tests/scalers/gcp/gcp_pubsub/gcp_pubsub_test.go similarity index 98% rename from tests/scalers/gcp_pubsub/gcp_pubsub_test.go rename to tests/scalers/gcp/gcp_pubsub/gcp_pubsub_test.go index d38fca3d019..cc69e222200 100644 --- a/tests/scalers/gcp_pubsub/gcp_pubsub_test.go +++ b/tests/scalers/gcp/gcp_pubsub/gcp_pubsub_test.go @@ -29,7 +29,7 @@ const ( ) var ( - gcpKey = os.Getenv("GCP_SP_KEY") + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") creds = make(map[string]interface{}) errGcpKey = json.Unmarshal([]byte(gcpKey), &creds) testNamespace = fmt.Sprintf("%s-ns", testName) @@ -168,7 +168,7 @@ spec: func TestScaler(t *testing.T) { // setup t.Log("--- setting up ---") - require.NotEmpty(t, gcpKey, "GCP_KEY env variable is required for GCP storage test") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") assert.NoErrorf(t, errGcpKey, "Failed to load credentials from gcpKey - %s", errGcpKey) // Create kubernetes resources diff --git a/tests/scalers/gcp/gcp_pubsub_workload_identity/gcp_pubsub_workload_identity_test.go b/tests/scalers/gcp/gcp_pubsub_workload_identity/gcp_pubsub_workload_identity_test.go new file mode 100644 index 00000000000..28ce57f55c5 --- /dev/null +++ b/tests/scalers/gcp/gcp_pubsub_workload_identity/gcp_pubsub_workload_identity_test.go @@ -0,0 +1,308 @@ +//go:build e2e +// +build e2e + +package gcp_pubsub_workload_identity_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +var now = time.Now().UnixNano() + +const ( + testName = "gcp-pubsub-workload-identity-test" +) + +var ( + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") + creds = make(map[string]interface{}) + errGcpKey = json.Unmarshal([]byte(gcpKey), &creds) + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + projectID = creds["project_id"] + topicID = fmt.Sprintf("projects/%s/topics/keda-test-topic-%d", projectID, now) + subscriptionName = fmt.Sprintf("keda-test-topic-sub-%d", now) + subscriptionID = fmt.Sprintf("projects/%s/subscriptions/%s", projectID, subscriptionName) + maxReplicaCount = 4 + activationThreshold = 5 + gsPrefix = fmt.Sprintf("kubectl exec --namespace %s deploy/gcp-sdk -- ", testNamespace) +) + +type templateData struct { + TestNamespace string + SecretName string + GcpCreds string + DeploymentName string + ScaledObjectName string + SubscriptionName string + SubscriptionID string + MaxReplicaCount int + ActivationThreshold int +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + creds.json: {{.GcpCreds}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}}-processor + image: google/cloud-sdk:slim + # Consume a message + command: [ "/bin/bash", "-c", "--" ] + args: [ "gcloud auth activate-service-account --key-file /etc/secret-volume/creds.json && \ + while true; do gcloud pubsub subscriptions pull {{.SubscriptionID}} --auto-ack; sleep 20; done" ] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS_JSON + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: creds.json + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + volumes: + - name: secret-volume + secret: + secretName: {{.SecretName}} +` + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials + namespace: {{.TestNamespace}} +spec: + podIdentity: + provider: gcp +` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: {{.MaxReplicaCount}} + cooldownPeriod: 10 + triggers: + - type: gcp-pubsub + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + subscriptionName: {{.SubscriptionName}} + mode: SubscriptionSize + value: "5" + activationValue: "{{.ActivationThreshold}}" +` + + gcpSdkTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gcp-sdk + namespace: {{.TestNamespace}} + labels: + app: gcp-sdk +spec: + replicas: 1 + selector: + matchLabels: + app: gcp-sdk + template: + metadata: + labels: + app: gcp-sdk + spec: + containers: + - name: gcp-sdk-container + image: google/cloud-sdk:slim + # Just spin & wait forever + command: [ "/bin/bash", "-c", "--" ] + args: [ "ls /tmp && while true; do sleep 30; done;" ] + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + volumes: + - name: secret-volume + secret: + secretName: {{.SecretName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") + assert.NoErrorf(t, errGcpKey, "Failed to load credentials from gcpKey - %s", errGcpKey) + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + sdkReady := WaitForDeploymentReplicaReadyCount(t, kc, "gcp-sdk", testNamespace, 1, 60, 1) + assert.True(t, sdkReady, "gcp-sdk deployment should be ready after a minute") + + if sdkReady { + if createPubsub(t) == nil { + // test scaling + testActivation(t, kc) + testScaleOut(t, kc) + testScaleIn(t, kc) + + // cleanup + t.Log("--- cleanup ---") + cleanupPubsub(t) + } + } + + DeleteKubernetesResources(t, kc, testNamespace, data, templates) +} + +func createPubsub(t *testing.T) error { + // Authenticate to GCP + t.Log("--- authenticate to GCP ---") + cmd := fmt.Sprintf("%sgcloud auth activate-service-account %s --key-file /etc/secret-volume/creds.json --project=%s", gsPrefix, creds["client_email"], projectID) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to set GCP authentication on gcp-sdk - %s", err) + if err != nil { + return err + } + + // Create topic + t.Log("--- create topic ---") + cmd = fmt.Sprintf("%sgcloud pubsub topics create %s", gsPrefix, topicID) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to create Pubsub topic %s: %s", topicID, err) + if err != nil { + return err + } + + // Create subscription + t.Log("--- create subscription ---") + cmd = fmt.Sprintf("%sgcloud pubsub subscriptions create %s --topic=%s", gsPrefix, subscriptionID, topicID) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to create Pubsub subscription %s: %s", subscriptionID, err) + + return err +} + +func cleanupPubsub(t *testing.T) { + // Delete the topic and subscription + t.Log("--- cleaning up the subscription and topic ---") + _, _ = ExecuteCommand(fmt.Sprintf("%sgcloud pubsub subscriptions delete %s", gsPrefix, subscriptionID)) + _, _ = ExecuteCommand(fmt.Sprintf("%sgcloud pubsub topics delete %s", gsPrefix, topicID)) +} + +func getTemplateData() (templateData, []Template) { + base64GcpCreds := base64.StdEncoding.EncodeToString([]byte(gcpKey)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + GcpCreds: base64GcpCreds, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + SubscriptionID: subscriptionID, + SubscriptionName: subscriptionName, + MaxReplicaCount: maxReplicaCount, + ActivationThreshold: activationThreshold, + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + {Name: "gcpSdkTemplate", Config: gcpSdkTemplate}, + } +} + +func publishMessages(t *testing.T, count int) { + t.Logf("--- publishing %d messages ---", count) + publish := fmt.Sprintf( + "%s/bin/bash -c -- 'for i in {1..%d}; do gcloud pubsub topics publish %s --message=AAAAAAAAAA;done'", + gsPrefix, + count, + topicID) + _, err := ExecuteCommand(publish) + assert.NoErrorf(t, err, "cannot publish messages to pubsub topic - %s", err) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing not scaling if below threshold ---") + + publishMessages(t, activationThreshold) + + t.Log("--- waiting to see replicas are not scaled up ---") + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 240) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale out ---") + + publishMessages(t, 20-activationThreshold) + + t.Log("--- waiting for replicas to scale out ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 30, 10), + fmt.Sprintf("replica count should be %d after five minutes", maxReplicaCount)) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale in ---") + cmd := fmt.Sprintf("%sgcloud pubsub subscriptions seek %s --time=-P1S", gsPrefix, subscriptionID) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "cannot reset subscription position - %s", err) + + t.Log("--- waiting for replicas to scale in to zero ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 30, 10), + "replica count should be 0 after five minute") +} diff --git a/tests/scalers/gcp_stackdriver/gcp_stackdriver_test.go b/tests/scalers/gcp/gcp_stackdriver/gcp_stackdriver_test.go similarity index 98% rename from tests/scalers/gcp_stackdriver/gcp_stackdriver_test.go rename to tests/scalers/gcp/gcp_stackdriver/gcp_stackdriver_test.go index b91d1d52062..7bd749714c8 100644 --- a/tests/scalers/gcp_stackdriver/gcp_stackdriver_test.go +++ b/tests/scalers/gcp/gcp_stackdriver/gcp_stackdriver_test.go @@ -29,7 +29,7 @@ const ( ) var ( - gcpKey = os.Getenv("GCP_SP_KEY") + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") creds = make(map[string]interface{}) errGcpKey = json.Unmarshal([]byte(gcpKey), &creds) testNamespace = fmt.Sprintf("%s-ns", testName) @@ -174,7 +174,7 @@ spec: func TestScaler(t *testing.T) { // setup t.Log("--- setting up ---") - require.NotEmpty(t, gcpKey, "GCP_KEY env variable is required for GCP storage test") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") assert.NoErrorf(t, errGcpKey, "Failed to load credentials from gcpKey - %s", errGcpKey) // Create kubernetes resources diff --git a/tests/scalers/gcp/gcp_stackdriver_workload_identity/gcp_stackdriver_workload_identity_test.go b/tests/scalers/gcp/gcp_stackdriver_workload_identity/gcp_stackdriver_workload_identity_test.go new file mode 100644 index 00000000000..32c931fc036 --- /dev/null +++ b/tests/scalers/gcp/gcp_stackdriver_workload_identity/gcp_stackdriver_workload_identity_test.go @@ -0,0 +1,315 @@ +//go:build e2e +// +build e2e + +package gcp_stackdriver_workload_identity_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "testing" + "time" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +var now = time.Now().UnixNano() + +const ( + testName = "gcp-stackdriver-workload-identity-test" +) + +var ( + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") + creds = make(map[string]interface{}) + errGcpKey = json.Unmarshal([]byte(gcpKey), &creds) + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + projectID = creds["project_id"] + topicName = fmt.Sprintf("keda-test-topic-%d", now) + topicID = fmt.Sprintf("projects/%s/topics/%s", projectID, topicName) + subscriptionName = fmt.Sprintf("keda-test-topic-sub-%d", now) + subscriptionID = fmt.Sprintf("projects/%s/subscriptions/%s", projectID, subscriptionName) + maxReplicaCount = 4 + activationThreshold = 5 + gsPrefix = fmt.Sprintf("kubectl exec --namespace %s deploy/gcp-sdk -- ", testNamespace) +) + +type templateData struct { + TestNamespace string + SecretName string + GcpCreds string + DeploymentName string + ScaledObjectName string + ProjectID string + TopicName string + SubscriptionName string + SubscriptionID string + MaxReplicaCount int + ActivationThreshold int +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + creds.json: {{.GcpCreds}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: {{.DeploymentName}}-processor + image: google/cloud-sdk:slim + # Consume a message + command: [ "/bin/bash", "-c", "--" ] + args: [ "gcloud auth activate-service-account --key-file /etc/secret-volume/creds.json && \ + while true; do gcloud pubsub subscriptions pull {{.SubscriptionID}} --auto-ack; sleep 20; done" ] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS_JSON + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: creds.json + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + volumes: + - name: secret-volume + secret: + secretName: {{.SecretName}} +` + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials + namespace: {{.TestNamespace}} +spec: + podIdentity: + provider: gcp` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: {{.MaxReplicaCount}} + cooldownPeriod: 10 + triggers: + - type: gcp-stackdriver + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + projectId: {{.ProjectID}} + filter: 'metric.type="pubsub.googleapis.com/topic/num_unacked_messages_by_region" AND resource.type="pubsub_topic" AND resource.label.topic_id="{{.TopicName}}"' + metricName: {{.TopicName}} + targetValue: "5" + activationTargetValue: "{{.ActivationThreshold}}" + alignmentPeriodSeconds: "60" + alignmentAligner: max +` + + gcpSdkTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gcp-sdk + namespace: {{.TestNamespace}} + labels: + app: gcp-sdk +spec: + replicas: 1 + selector: + matchLabels: + app: gcp-sdk + template: + metadata: + labels: + app: gcp-sdk + spec: + containers: + - name: gcp-sdk-container + image: google/cloud-sdk:slim + # Just spin & wait forever + command: [ "/bin/bash", "-c", "--" ] + args: [ "ls /tmp && while true; do sleep 30; done;" ] + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + volumes: + - name: secret-volume + secret: + secretName: {{.SecretName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") + assert.NoErrorf(t, errGcpKey, "Failed to load credentials from gcpKey - %s", errGcpKey) + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after a minute") + + sdkReady := WaitForDeploymentReplicaReadyCount(t, kc, "gcp-sdk", testNamespace, 1, 60, 1) + assert.True(t, sdkReady, "gcp-sdk deployment should be ready after a minute") + + if sdkReady { + if createPubsub(t) == nil { + // test scaling + testActivation(t, kc) + testScaleOut(t, kc) + testScaleIn(t, kc) + + // cleanup + t.Log("--- cleanup ---") + cleanupPubsub(t) + } + } + + DeleteKubernetesResources(t, kc, testNamespace, data, templates) +} + +func createPubsub(t *testing.T) error { + // Authenticate to GCP + t.Log("--- authenticate to GCP ---") + cmd := fmt.Sprintf("%sgcloud auth activate-service-account %s --key-file /etc/secret-volume/creds.json --project=%s", gsPrefix, creds["client_email"], projectID) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to set GCP authentication on gcp-sdk - %s", err) + if err != nil { + return err + } + + // Create topic + t.Log("--- create topic ---") + cmd = fmt.Sprintf("%sgcloud pubsub topics create %s", gsPrefix, topicID) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to create Pubsub topic %s: %s", topicID, err) + if err != nil { + return err + } + + // Create subscription + t.Log("--- create subscription ---") + cmd = fmt.Sprintf("%sgcloud pubsub subscriptions create %s --topic=%s", gsPrefix, subscriptionID, topicID) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to create Pubsub subscription %s: %s", subscriptionID, err) + + return err +} + +func cleanupPubsub(t *testing.T) { + // Delete the topic and subscription + t.Log("--- cleaning up the subscription and topic ---") + _, _ = ExecuteCommand(fmt.Sprintf("%sgcloud pubsub subscriptions delete %s", gsPrefix, subscriptionID)) + _, _ = ExecuteCommand(fmt.Sprintf("%sgcloud pubsub topics delete %s", gsPrefix, topicID)) +} + +func getTemplateData() (templateData, []Template) { + base64GcpCreds := base64.StdEncoding.EncodeToString([]byte(gcpKey)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + GcpCreds: base64GcpCreds, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + ProjectID: fmt.Sprintf("%s", projectID), + TopicName: topicName, + SubscriptionID: subscriptionID, + SubscriptionName: subscriptionName, + MaxReplicaCount: maxReplicaCount, + ActivationThreshold: activationThreshold, + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + {Name: "gcpSdkTemplate", Config: gcpSdkTemplate}, + } +} + +func publishMessages(t *testing.T, count int) { + t.Logf("--- publishing %d messages ---", count) + publish := fmt.Sprintf( + "%s/bin/bash -c -- 'for i in {1..%d}; do gcloud pubsub topics publish %s --message=AAAAAAAAAA;done'", + gsPrefix, + count, + topicID) + _, err := ExecuteCommand(publish) + assert.NoErrorf(t, err, "cannot publish messages to pubsub topic - %s", err) +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing not scaling if below threshold ---") + + publishMessages(t, activationThreshold) + + t.Log("--- waiting to see replicas are not scaled up ---") + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 240) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale out ---") + + publishMessages(t, 20-activationThreshold) + + t.Log("--- waiting for replicas to scale out ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 30, 10), + fmt.Sprintf("replica count should be %d after five minutes", maxReplicaCount)) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale in ---") + cmd := fmt.Sprintf("%sgcloud pubsub subscriptions seek %s --time=-P1S", gsPrefix, subscriptionID) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "cannot reset subscription position - %s", err) + + t.Log("--- waiting for replicas to scale in to zero ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 30, 10), + "replica count should be 0 after five minute") +} diff --git a/tests/scalers/gcp_storage/gcp_storage_test.go b/tests/scalers/gcp/gcp_storage/gcp_storage_test.go similarity index 97% rename from tests/scalers/gcp_storage/gcp_storage_test.go rename to tests/scalers/gcp/gcp_storage/gcp_storage_test.go index 12d50b645b9..53f52b3d37f 100644 --- a/tests/scalers/gcp_storage/gcp_storage_test.go +++ b/tests/scalers/gcp/gcp_storage/gcp_storage_test.go @@ -26,7 +26,7 @@ const ( ) var ( - gcpKey = os.Getenv("GCP_SP_KEY") + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") testNamespace = fmt.Sprintf("%s-ns", testName) secretName = fmt.Sprintf("%s-secret", testName) deploymentName = fmt.Sprintf("%s-deployment", testName) @@ -149,7 +149,7 @@ spec: func TestScaler(t *testing.T) { // setup t.Log("--- setting up ---") - require.NotEmpty(t, gcpKey, "GCP_KEY env variable is required for GCP storage test") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") // Create kubernetes resources kc := GetKubernetesClient(t) diff --git a/tests/scalers/gcp/gcp_storage_workload_identity/gcp_storage_workload_identity_test.go b/tests/scalers/gcp/gcp_storage_workload_identity/gcp_storage_workload_identity_test.go new file mode 100644 index 00000000000..e57d9cd52bd --- /dev/null +++ b/tests/scalers/gcp/gcp_storage_workload_identity/gcp_storage_workload_identity_test.go @@ -0,0 +1,276 @@ +//go:build e2e +// +build e2e + +package gcp_storage_workload_identity_test + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "testing" + + "github.com/joho/godotenv" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/client-go/kubernetes" + + . "github.com/kedacore/keda/v2/tests/helper" +) + +// Load environment variables from .env file +var _ = godotenv.Load("../../.env") + +const ( + testName = "gcp-storage-workload-identity-test" +) + +var ( + gcpKey = os.Getenv("TF_GCP_SA_CREDENTIALS") + testNamespace = fmt.Sprintf("%s-ns", testName) + secretName = fmt.Sprintf("%s-secret", testName) + deploymentName = fmt.Sprintf("%s-deployment", testName) + scaledObjectName = fmt.Sprintf("%s-so", testName) + bucketName = fmt.Sprintf("%s-bucket", testName) + maxReplicaCount = 3 + activationThreshold = 5 + gsPrefix = fmt.Sprintf("kubectl exec --namespace %s deploy/gcp-sdk -- ", testNamespace) +) + +type templateData struct { + TestNamespace string + SecretName string + GcpCreds string + DeploymentName string + ScaledObjectName string + BucketName string + MaxReplicaCount int + ActivationThreshold int +} + +const ( + secretTemplate = ` +apiVersion: v1 +kind: Secret +metadata: + name: {{.SecretName}} + namespace: {{.TestNamespace}} +data: + creds.json: {{.GcpCreds}} +` + + deploymentTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{.DeploymentName}} + namespace: {{.TestNamespace}} + labels: + app: {{.DeploymentName}} +spec: + replicas: 0 + selector: + matchLabels: + app: {{.DeploymentName}} + template: + metadata: + labels: + app: {{.DeploymentName}} + spec: + containers: + - name: noop-processor + image: ubuntu:20.04 + command: ["/bin/bash"] + args: ["-c", "sleep 60"] + env: + - name: GOOGLE_APPLICATION_CREDENTIALS_JSON + valueFrom: + secretKeyRef: + name: {{.SecretName}} + key: creds.json +` + + triggerAuthenticationTemplate = `apiVersion: keda.sh/v1alpha1 +kind: TriggerAuthentication +metadata: + name: keda-trigger-auth-gcp-credentials + namespace: {{.TestNamespace}} +spec: + podIdentity: + provider: gcp` + + scaledObjectTemplate = ` +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{.ScaledObjectName}} + namespace: {{.TestNamespace}} +spec: + scaleTargetRef: + name: {{.DeploymentName}} + pollingInterval: 5 + minReplicaCount: 0 + maxReplicaCount: {{.MaxReplicaCount}} + cooldownPeriod: 10 + triggers: + - type: gcp-storage + authenticationRef: + name: keda-trigger-auth-gcp-credentials + metadata: + bucketName: {{.BucketName}} + targetObjectCount: '5' + activationTargetObjectCount: '{{.ActivationThreshold}}' +` + + gcpSdkTemplate = ` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gcp-sdk + namespace: {{.TestNamespace}} + labels: + app: gcp-sdk +spec: + replicas: 1 + selector: + matchLabels: + app: gcp-sdk + template: + metadata: + labels: + app: gcp-sdk + spec: + containers: + - name: gcp-sdk-container + image: google/cloud-sdk:slim + # Just spin & wait forever + command: [ "/bin/bash", "-c", "--" ] + args: [ "ls /tmp && while true; do sleep 30; done;" ] + volumeMounts: + - name: secret-volume + mountPath: /etc/secret-volume + volumes: + - name: secret-volume + secret: + secretName: {{.SecretName}} +` +) + +func TestScaler(t *testing.T) { + // setup + t.Log("--- setting up ---") + require.NotEmpty(t, gcpKey, "TF_GCP_SA_CREDENTIALS env variable is required for GCP storage test") + + // Create kubernetes resources + kc := GetKubernetesClient(t) + data, templates := getTemplateData() + + CreateKubernetesResources(t, kc, testNamespace, data, templates) + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 60, 1), + "replica count should be 0 after 1 minute") + + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, "gcp-sdk", testNamespace, 1, 60, 1), + "gcp-sdk deployment should be ready after 1 minute") + + if createBucket(t) == nil { + // test scaling + testActivation(t, kc) + testScaleOut(t, kc) + testScaleIn(t, kc) + } + + // cleanup + t.Log("--- cleanup ---") + cleanupBucket(t) + DeleteKubernetesResources(t, kc, testNamespace, data, templates) +} + +func createBucket(t *testing.T) error { + // Authenticate to GCP + creds := make(map[string]interface{}) + err := json.Unmarshal([]byte(gcpKey), &creds) + assert.NoErrorf(t, err, "Failed to load credentials from gcpKey - %s", err) + + cmd := fmt.Sprintf("%sgcloud auth activate-service-account %s --key-file /etc/secret-volume/creds.json --project=%s", gsPrefix, creds["client_email"], creds["project_id"]) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to set GCP authentication on gcp-sdk - %s", err) + + cleanupBucket(t) + + // Create bucket + cmd = fmt.Sprintf("%sgsutil mb gs://%s", gsPrefix, bucketName) + _, err = ExecuteCommand(cmd) + assert.NoErrorf(t, err, "Failed to create GCS bucket - %s", err) + return err +} + +func cleanupBucket(t *testing.T) { + // Cleanup the bucket + t.Log("--- cleaning up the bucket ---") + _, _ = ExecuteCommand(fmt.Sprintf("%sgsutil -m rm -r gs://%s", gsPrefix, bucketName)) +} + +func getTemplateData() (templateData, []Template) { + base64GcpCreds := base64.StdEncoding.EncodeToString([]byte(gcpKey)) + + return templateData{ + TestNamespace: testNamespace, + SecretName: secretName, + GcpCreds: base64GcpCreds, + DeploymentName: deploymentName, + ScaledObjectName: scaledObjectName, + BucketName: bucketName, + MaxReplicaCount: maxReplicaCount, + ActivationThreshold: activationThreshold, + }, []Template{ + {Name: "secretTemplate", Config: secretTemplate}, + {Name: "deploymentTemplate", Config: deploymentTemplate}, + {Name: "triggerAuthenticationTemplate", Config: triggerAuthenticationTemplate}, + {Name: "scaledObjectTemplate", Config: scaledObjectTemplate}, + {Name: "gcpSdkTemplate", Config: gcpSdkTemplate}, + } +} + +func uploadFiles(t *testing.T, prefix string, count int) { + t.Logf("--- uploading %d files ---", count) + + for i := 0; i < count; i++ { + cmd := fmt.Sprintf("%sgsutil cp -n /usr/lib/google-cloud-sdk/bin/gsutil gs://%s/gsutil-%s%d", gsPrefix, bucketName, prefix, i) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "cannot upload file to bucket - %s", err) + } +} + +func testActivation(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing not scaling if below threshold ---") + + uploadFiles(t, "active", activationThreshold) + + t.Log("--- waiting to see replicas are not scaled up ---") + AssertReplicaCountNotChangeDuringTimePeriod(t, kc, deploymentName, testNamespace, 0, 240) +} + +func testScaleOut(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale out ---") + + uploadFiles(t, "scaling", 30-activationThreshold) + + t.Log("--- waiting for replicas to scale out ---") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, maxReplicaCount, 60, 5), + fmt.Sprintf("replica count should be %d after five minutes", maxReplicaCount)) +} + +func testScaleIn(t *testing.T, kc *kubernetes.Clientset) { + t.Log("--- testing scale in ---") + + // Delete files so we are still left with activationThreshold number of files which should be enough + // to scale in to 0. + cmd := fmt.Sprintf("%sgsutil -m rm -a gs://%s/gsutil*", gsPrefix, bucketName) + _, err := ExecuteCommand(cmd) + assert.NoErrorf(t, err, "cannot clear bucket - %s", err) + + t.Log("--- waiting for replicas to scale in to zero") + assert.True(t, WaitForDeploymentReplicaReadyCount(t, kc, deploymentName, testNamespace, 0, 30, 10), + "replica count should be 0 after 5 minutes") +} diff --git a/tests/utils/cleanup_test.go b/tests/utils/cleanup_test.go index 8ce1b037d8e..efb3d44e53e 100644 --- a/tests/utils/cleanup_test.go +++ b/tests/utils/cleanup_test.go @@ -41,10 +41,33 @@ func TestRemoveAwsIdentityComponents(t *testing.T) { _, err := ExecuteCommand(fmt.Sprintf("helm uninstall aws-identity-webhook --namespace %s", AwsIdentityNamespace)) require.NoErrorf(t, err, "cannot uninstall workload identity webhook - %s", err) - _, err = ExecuteCommand(fmt.Sprintf("helm uninstall cert-manager --namespace %s", AwsIdentityNamespace)) + KubeClient = GetKubernetesClient(t) + + DeleteNamespace(t, KubeClient, AwsIdentityNamespace) +} + +func TestRemoveGcpIdentityComponents(t *testing.T) { + if GcpIdentityTests == "" || GcpIdentityTests == StringFalse { + t.Skip("skipping as workload identity tests are disabled") + } + + _, err := ExecuteCommand(fmt.Sprintf("helm uninstall gcp-identity-webhook --namespace %s", GcpIdentityNamespace)) + require.NoErrorf(t, err, "cannot uninstall workload identity webhook - %s", err) + + KubeClient = GetKubernetesClient(t) + + DeleteNamespace(t, KubeClient, GcpIdentityNamespace) +} + +func TestRemoveCertManager(t *testing.T) { + if !InstallCertManager { + t.Skip("skipping as cert manager isn't required") + } + + _, err := ExecuteCommand(fmt.Sprintf("helm uninstall cert-manager --namespace %s", CertManagerNamespace)) require.NoErrorf(t, err, "cannot uninstall cert-manager - %s", err) KubeClient = GetKubernetesClient(t) - DeleteNamespace(t, KubeClient, AwsIdentityNamespace) + DeleteNamespace(t, KubeClient, CertManagerNamespace) } diff --git a/tests/utils/setup_test.go b/tests/utils/setup_test.go index ef191261b5b..ef6e680f3dd 100644 --- a/tests/utils/setup_test.go +++ b/tests/utils/setup_test.go @@ -97,6 +97,28 @@ func TestSetupWorkloadIdentityComponents(t *testing.T) { require.True(t, success, "expected workload identity webhook deployment to start 2 pods successfully") } +func TestSetupCertManager(t *testing.T) { + if !InstallCertManager { + t.Skip("skipping cert manager is not required") + } + + _, err := ExecuteCommand("helm version") + require.NoErrorf(t, err, "helm is not installed - %s", err) + + _, err = ExecuteCommand("helm repo add jetstack https://charts.jetstack.io") + require.NoErrorf(t, err, "cannot add jetstack helm repo - %s", err) + + _, err = ExecuteCommand("helm repo update jetstack") + require.NoErrorf(t, err, "cannot update jetstack helm repo - %s", err) + + KubeClient = GetKubernetesClient(t) + CreateNamespace(t, KubeClient, CertManagerNamespace) + + _, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install cert-manager jetstack/cert-manager --namespace %s --set installCRDs=true --wait", + CertManagerNamespace)) + require.NoErrorf(t, err, "cannot install cert-manager - %s", err) +} + func TestSetupAwsIdentityComponents(t *testing.T) { if AwsIdentityTests == "" || AwsIdentityTests == StringFalse { t.Skip("skipping aws identity tests are disabled") @@ -111,23 +133,36 @@ func TestSetupAwsIdentityComponents(t *testing.T) { _, err = ExecuteCommand("helm repo update jkroepke") require.NoErrorf(t, err, "cannot update jkroepke helm repo - %s", err) - _, err = ExecuteCommand("helm repo add jetstack https://charts.jetstack.io") - require.NoErrorf(t, err, "cannot add jetstack helm repo - %s", err) - - _, err = ExecuteCommand("helm repo update jetstack") - require.NoErrorf(t, err, "cannot update jetstack helm repo - %s", err) - KubeClient = GetKubernetesClient(t) CreateNamespace(t, KubeClient, AwsIdentityNamespace) - _, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install cert-manager jetstack/cert-manager --namespace %s --set installCRDs=true --wait", - AwsIdentityNamespace)) - require.NoErrorf(t, err, "cannot install cert-manager - %s", err) - _, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install aws-identity-webhook jkroepke/amazon-eks-pod-identity-webhook --namespace %s --set fullnameOverride=aws-identity-webhook --wait", AwsIdentityNamespace)) require.NoErrorf(t, err, "cannot install workload identity webhook - %s", err) - time.Sleep(2 * time.Minute) // sleep for some time for webhook to setup properly + time.Sleep(1 * time.Minute) // sleep for some time for webhook to setup properly +} + +func TestSetupGcpIdentityComponents(t *testing.T) { + if GcpIdentityTests == "" || GcpIdentityTests == StringFalse { + t.Skip("skipping gcp identity tests are disabled") + } + + _, err := ExecuteCommand("helm version") + require.NoErrorf(t, err, "helm is not installed - %s", err) + + _, err = ExecuteCommand("helm repo add gcp-workload-identity-federation-webhook https://pfnet-research.github.io/gcp-workload-identity-federation-webhook") + require.NoErrorf(t, err, "cannot add gcp-workload-identity-federation-webhook helm repo - %s", err) + + _, err = ExecuteCommand("helm repo update gcp-workload-identity-federation-webhook") + require.NoErrorf(t, err, "cannot update gcp-workload-identity-federation-webhook helm repo - %s", err) + + KubeClient = GetKubernetesClient(t) + CreateNamespace(t, KubeClient, GcpIdentityNamespace) + + _, err = ExecuteCommand(fmt.Sprintf("helm upgrade --install gcp-identity-webhook gcp-workload-identity-federation-webhook/gcp-workload-identity-federation-webhook --namespace %s --set fullnameOverride=gcp-identity-webhook --set controllerManager.manager.args[0]=--token-default-mode=0444 --wait", + GcpIdentityNamespace)) + require.NoErrorf(t, err, "cannot install workload identity webhook - %s", err) + time.Sleep(1 * time.Minute) // sleep for some time for webhook to setup properly } func TestDeployKEDA(t *testing.T) {