From 4059835367a69a30d060ef005d8c13dffa2c6af4 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 5 Dec 2024 09:49:04 +0100 Subject: [PATCH 01/44] Add ingress test for opaAuthorizeRequest filter --- test/e2e/apply/secret.yaml | 1 + test/e2e/cluster_config.sh | 2 ++ test/e2e/ingress.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/test/e2e/apply/secret.yaml b/test/e2e/apply/secret.yaml index aeda315268..37c5c13c95 100644 --- a/test/e2e/apply/secret.yaml +++ b/test/e2e/apply/secret.yaml @@ -19,3 +19,4 @@ data: ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu" OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN" SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ==" + STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A==" diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index 9bcee96c8f..94121a3988 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -47,6 +47,8 @@ clusters: karpenter_pools_enabled: "true" okta_auth_client_id: "kubernetes.cluster.teapot-e2e" teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" + skipper_open_policy_agent_enabled: "true" + skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 93cab4e5c6..5edc9b0ac6 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -266,6 +266,35 @@ var __ = describe("Ingress tests simple", func() { framework.ExpectNoError(err) Expect(s).To(Equal(backendContent)) + // Test opaAuthorizeRequest filter in ingress + path = "/" + opaPolicyName := "styra-smoketest" + updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + netv1.PathTypeImplementationSpecific, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + }, + port, + ) + ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(10 * time.Second) // wait for routing change propagation + + req.Header.Set("Authorization", "Basic valid_token") + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + // Test additional hostname additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) addHostIng := addHostIngress(updatedIng, additionalHostname) From ee111a019e5579b8373c786cd437d9e7a7eb3eec Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 5 Dec 2024 13:02:02 +0100 Subject: [PATCH 02/44] Include styra token in env --- delivery.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/delivery.yaml b/delivery.yaml index 91b8e61804..0d15788058 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -155,6 +155,11 @@ pipeline: secretKeyRef: name: kubernetes-e2e-config-secret key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" - name: CLUSTER_ADMIN_TOKEN valueFrom: secretKeyRef: From 983c7b900f5c77b5d4e685d75831f80e10a4d884 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 5 Dec 2024 14:11:59 +0100 Subject: [PATCH 03/44] Provide bucket arn --- test/e2e/cluster_config.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index 94121a3988..cc001cf535 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -49,6 +49,7 @@ clusters: teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" skipper_open_policy_agent_enabled: "true" skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" + skipper_open_policy_agent_bucket_arn: "arn:aws:s3:::zalando-open-policy-agent-sandbox-bundles" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} From eb483bd72587164f28d198caffd09ff598ddcee6 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 5 Dec 2024 14:56:02 +0100 Subject: [PATCH 04/44] Provide more environment variables --- test/e2e/cluster_config.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index cc001cf535..72a6d110af 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -50,6 +50,8 @@ clusters: skipper_open_policy_agent_enabled: "true" skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" skipper_open_policy_agent_bucket_arn: "arn:aws:s3:::zalando-open-policy-agent-sandbox-bundles" + skipper_open_policy_agent_observability_url: "https://open-policy-agent-sandbox-observability.corporate-iam.zalan.do/v1" + skipper_open_policy_agent_bundles_url: "https://zalando-open-policy-agent-sandbox-bundles.s3.eu-central-1.amazonaws.com" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} From 56c00ba08e4c705fb49ea28ae34a09a16dee8b0c Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 5 Dec 2024 16:04:38 +0100 Subject: [PATCH 05/44] Add failing test --- test/e2e/ingress.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 5edc9b0ac6..a4629e1023 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -266,7 +266,7 @@ var __ = describe("Ingress tests simple", func() { framework.ExpectNoError(err) Expect(s).To(Equal(backendContent)) - // Test opaAuthorizeRequest filter in ingress + // Test ingress Filters: opaAuthorizeRequest path = "/" opaPolicyName := "styra-smoketest" updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, @@ -286,15 +286,19 @@ var __ = describe("Ingress tests simple", func() { By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) time.Sleep(10 * time.Second) // wait for routing change propagation - req.Header.Set("Authorization", "Basic valid_token") + req.Header.Set("Authorization", "Basic valid_token") //Authorized request resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) framework.ExpectNoError(err) Expect(resp.StatusCode).To(Equal(http.StatusOK)) - Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) s, err = getBody(resp) framework.ExpectNoError(err) Expect(s).To(Equal(backendContent)) + req.Header.Set("Authorization", "Basic invalid_token") //Unauthorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) //Intentionally failing the test for once + // Test additional hostname additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) addHostIng := addHostIngress(updatedIng, additionalHostname) From 453a65170f595cf14fd31b15085a96e46d20c099 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 07:54:05 +0100 Subject: [PATCH 06/44] Focus on ingress tests --- test/e2e/run_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index eb8b2f5538..9ba67f4e32 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ + -focus="(\[[Ingress]\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From be209f0b8151bc7fd37c2d9634ac48aa968e5790 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 08:32:25 +0100 Subject: [PATCH 07/44] Focus on ingress tests --- test/e2e/run_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 9ba67f4e32..654b1c4967 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[[Ingress]\])" \ + -focus="(\[Ingress\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From 07b98d3390890494e52005d88dd8235382daeadc Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 09:28:51 +0100 Subject: [PATCH 08/44] Comment failing tests --- test/e2e/ingress.go | 90 +++++++++++++++++++++++---------------------- test/e2e/run_e2e.sh | 2 +- 2 files changed, 47 insertions(+), 45 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index a4629e1023..05804a04f8 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -124,7 +124,7 @@ var __ = describe("Ingress tests simple", func() { jig *ingress.TestJig ) - It("Should create simple ingress [Ingress]", func() { + It("Should create simple ingress [IngressOpa]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" @@ -294,57 +294,59 @@ var __ = describe("Ingress tests simple", func() { framework.ExpectNoError(err) Expect(s).To(Equal(backendContent)) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(10 * time.Second) req.Header.Set("Authorization", "Basic invalid_token") //Unauthorized request resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) framework.ExpectNoError(err) Expect(resp.StatusCode).To(Equal(http.StatusOK)) //Intentionally failing the test for once // Test additional hostname - additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) - addHostIng := addHostIngress(updatedIng, additionalHostname) - ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), addHostIng, metav1.UpdateOptions{}) - framework.ExpectNoError(err) - By("Waiting for new DNS hostname to be resolvable " + additionalHostname) - err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) - framework.ExpectNoError(err) - By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) - By(fmt.Sprintf("Testing the new hostname %s for ingress %s/%s we make sure old routes are working", additionalHostname, ingressUpdate.Namespace, ingressUpdate.Name)) - url = "https://" + additionalHostname + "/" - req, err = http.NewRequest("GET", url, nil) - framework.ExpectNoError(err) - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) + //additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) + //addHostIng := addHostIngress(updatedIng, additionalHostname) + //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), addHostIng, metav1.UpdateOptions{}) + //framework.ExpectNoError(err) + //By("Waiting for new DNS hostname to be resolvable " + additionalHostname) + //err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) + //framework.ExpectNoError(err) + //By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) + //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusOK)) + //s, err = getBody(resp) + //framework.ExpectNoError(err) + //Expect(s).To(Equal(backendContent)) + //By(fmt.Sprintf("Testing the new hostname %s for ingress %s/%s we make sure old routes are working", additionalHostname, ingressUpdate.Namespace, ingressUpdate.Name)) + //url = "https://" + additionalHostname + "/" + //req, err = http.NewRequest("GET", url, nil) + //framework.ExpectNoError(err) + //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusOK)) + //s, err = getBody(resp) + //framework.ExpectNoError(err) + //Expect(s).To(Equal(backendContent)) // Test changed path - newPath := "/foo" - changePathIng := changePathIngress(updatedIng, newPath) - ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), changePathIng, metav1.UpdateOptions{}) - framework.ExpectNoError(err) - - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the old request, because of the path route", ingressUpdate.Namespace, ingressUpdate.Name)) - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) - pathURL := "https://" + hostName + newPath - pathReq, err := http.NewRequest("GET", pathURL, nil) - framework.ExpectNoError(err) - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 for a new request to the path route", ingressUpdate.Namespace, ingressUpdate.Name)) - resp, err = getAndWaitResponse(rt, pathReq, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) + //newPath := "/foo" + //changePathIng := changePathIngress(updatedIng, newPath) + //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), changePathIng, metav1.UpdateOptions{}) + //framework.ExpectNoError(err) + // + //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the old request, because of the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + //pathURL := "https://" + hostName + newPath + //pathReq, err := http.NewRequest("GET", pathURL, nil) + //framework.ExpectNoError(err) + //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 for a new request to the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + //resp, err = getAndWaitResponse(rt, pathReq, 10*time.Second, http.StatusOK) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusOK)) + //s, err = getBody(resp) + //framework.ExpectNoError(err) + //Expect(s).To(Equal(backendContent)) }) }) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 654b1c4967..bf75f441fc 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Ingress\])" \ + -focus="(\[IngressOpa\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From 3b58b10998c5e5ec0e32bbfd8496da0239e39617 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 10:37:19 +0100 Subject: [PATCH 09/44] Run single failing test --- test/e2e/ingress.go | 66 ++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 05804a04f8..654ba4666d 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -239,32 +239,32 @@ var __ = describe("Ingress tests simple", func() { Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) // Test ingress Filters - path = "/" - headerKey := "X-Foo" - headerVal := "f00" - updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, - ingressCreate.ObjectMeta.Namespace, - hostName, - serviceName, - path, - netv1.PathTypeImplementationSpecific, - ingressCreate.ObjectMeta.Labels, - map[string]string{ - "zalando.org/skipper-filter": fmt.Sprintf(`setResponseHeader("%s", "%s")`, headerKey, headerVal), - }, - port, - ) - ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) - framework.ExpectNoError(err) - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) - time.Sleep(10 * time.Second) // wait for routing change propagation - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) + //path = "/" + //headerKey := "X-Foo" + //headerVal := "f00" + //updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, + // ingressCreate.ObjectMeta.Namespace, + // hostName, + // serviceName, + // path, + // netv1.PathTypeImplementationSpecific, + // ingressCreate.ObjectMeta.Labels, + // map[string]string{ + // "zalando.org/skipper-filter": fmt.Sprintf(`setResponseHeader("%s", "%s")`, headerKey, headerVal), + // }, + // port, + //) + //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + //framework.ExpectNoError(err) + //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) + //time.Sleep(10 * time.Second) // wait for routing change propagation + //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusOK)) + //Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) + //s, err = getBody(resp) + //framework.ExpectNoError(err) + //Expect(s).To(Equal(backendContent)) // Test ingress Filters: opaAuthorizeRequest path = "/" @@ -286,13 +286,13 @@ var __ = describe("Ingress tests simple", func() { By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) time.Sleep(10 * time.Second) // wait for routing change propagation - req.Header.Set("Authorization", "Basic valid_token") //Authorized request - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) + //req.Header.Set("Authorization", "Basic valid_token") //Authorized request + //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + //framework.ExpectNoError(err) + //Expect(resp.StatusCode).To(Equal(http.StatusOK)) + //s, err = getBody(resp) + //framework.ExpectNoError(err) + //Expect(s).To(Equal(backendContent)) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) time.Sleep(10 * time.Second) From a240bbc7763d0af5a8ec63d2994305979d17c63a Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 16:16:45 +0100 Subject: [PATCH 10/44] Separate test for OPA filter --- test/e2e/ingress.go | 278 +++++++++++++++++++++++++++++--------------- test/e2e/run_e2e.sh | 2 +- 2 files changed, 186 insertions(+), 94 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 654ba4666d..c216d00768 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -124,7 +124,7 @@ var __ = describe("Ingress tests simple", func() { jig *ingress.TestJig ) - It("Should create simple ingress [IngressOpa]", func() { + It("Should create simple ingress [Ingress]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" @@ -239,36 +239,9 @@ var __ = describe("Ingress tests simple", func() { Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) // Test ingress Filters - //path = "/" - //headerKey := "X-Foo" - //headerVal := "f00" - //updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, - // ingressCreate.ObjectMeta.Namespace, - // hostName, - // serviceName, - // path, - // netv1.PathTypeImplementationSpecific, - // ingressCreate.ObjectMeta.Labels, - // map[string]string{ - // "zalando.org/skipper-filter": fmt.Sprintf(`setResponseHeader("%s", "%s")`, headerKey, headerVal), - // }, - // port, - //) - //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) - //framework.ExpectNoError(err) - //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) - //time.Sleep(10 * time.Second) // wait for routing change propagation - //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusOK)) - //Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) - //s, err = getBody(resp) - //framework.ExpectNoError(err) - //Expect(s).To(Equal(backendContent)) - - // Test ingress Filters: opaAuthorizeRequest path = "/" - opaPolicyName := "styra-smoketest" + headerKey := "X-Foo" + headerVal := "f00" updatedIng = updateIngress(ingressCreate.ObjectMeta.Name, ingressCreate.ObjectMeta.Namespace, hostName, @@ -277,80 +250,199 @@ var __ = describe("Ingress tests simple", func() { netv1.PathTypeImplementationSpecific, ingressCreate.ObjectMeta.Labels, map[string]string{ - "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + "zalando.org/skipper-filter": fmt.Sprintf(`setResponseHeader("%s", "%s")`, headerKey, headerVal), }, port, ) ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) framework.ExpectNoError(err) - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with %s header set to %s for the next request", ingressUpdate.Namespace, ingressUpdate.Name, headerKey, headerVal)) time.Sleep(10 * time.Second) // wait for routing change propagation + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + Expect(resp.Header.Get(headerKey)).To(Equal(headerVal)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + + // Test additional hostname + additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) + addHostIng := addHostIngress(updatedIng, additionalHostname) + ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), addHostIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + By("Waiting for new DNS hostname to be resolvable " + additionalHostname) + err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) + framework.ExpectNoError(err) + By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + By(fmt.Sprintf("Testing the new hostname %s for ingress %s/%s we make sure old routes are working", additionalHostname, ingressUpdate.Namespace, ingressUpdate.Name)) + url = "https://" + additionalHostname + "/" + req, err = http.NewRequest("GET", url, nil) + framework.ExpectNoError(err) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + + // Test changed path + newPath := "/foo" + changePathIng := changePathIngress(updatedIng, newPath) + ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), changePathIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the old request, because of the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) + pathURL := "https://" + hostName + newPath + pathReq, err := http.NewRequest("GET", pathURL, nil) + framework.ExpectNoError(err) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 for a new request to the path route", ingressUpdate.Namespace, ingressUpdate.Name)) + resp, err = getAndWaitResponse(rt, pathReq, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + }) +}) + +var ___ = describe("Ingress tests for OPA filters", func() { + f := framework.NewDefaultFramework("skipper-ingress-with-opa") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + var ( + cs kubernetes.Interface + jig *ingress.TestJig + ) + + It("Should activate OPA filter without issue [Ingress] [Zalando1]", func() { + jig = ingress.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "mytest" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + waitTime := 10 * time.Minute + + // CREATE setup + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + _, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port) + ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime) + framework.ExpectNoError(err) + + _, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + framework.ExpectNoError(err) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + framework.ExpectNoError(err) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + framework.ExpectNoError(err) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + framework.ExpectNoError(err) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + framework.ExpectNoError(err) + resp, err := rt.RoundTrip(req) + framework.ExpectNoError(err) + s, err := getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) - //req.Header.Set("Authorization", "Basic valid_token") //Authorized request - //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusOK)) - //s, err = getBody(resp) - //framework.ExpectNoError(err) - //Expect(s).To(Equal(backendContent)) + // Start actual ingress tests + // Test ingress Filters: opaAuthorizeRequest + + /** + ## The Rule looks like below. + ## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego + default allow := false + + allow if { + input.attributes.request.http.method == "GET" + auth_header_val := input.attributes.request.http.headers.authorization + startswith(auth_header_val, "Basic ") + token := substring(auth_header_val, count("Basic "), -1) + token == "valid_token" + } + */ + path := "/" + opaPolicyName := "styra-smoketest" + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + netv1.PathTypeImplementationSpecific, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + }, + port, + ) + ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(10 * time.Second) // wait for routing change propagation + req.Header.Set("Authorization", "Basic valid_token") //Authorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) - time.Sleep(10 * time.Second) req.Header.Set("Authorization", "Basic invalid_token") //Unauthorized request resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) //Intentionally failing the test for once - - // Test additional hostname - //additionalHostname := fmt.Sprintf("foo-%d.%s", time.Now().UTC().Unix(), E2EHostedZone()) - //addHostIng := addHostIngress(updatedIng, additionalHostname) - //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), addHostIng, metav1.UpdateOptions{}) - //framework.ExpectNoError(err) - //By("Waiting for new DNS hostname to be resolvable " + additionalHostname) - //err = waitForResponse(additionalHostname, "https", waitTime, isSuccess, false) - //framework.ExpectNoError(err) - //By(fmt.Sprintf("Testing the old hostname %s for ingress %s/%s we make sure old routes are working", hostName, ingressUpdate.Namespace, ingressUpdate.Name)) - //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusOK)) - //s, err = getBody(resp) - //framework.ExpectNoError(err) - //Expect(s).To(Equal(backendContent)) - //By(fmt.Sprintf("Testing the new hostname %s for ingress %s/%s we make sure old routes are working", additionalHostname, ingressUpdate.Namespace, ingressUpdate.Name)) - //url = "https://" + additionalHostname + "/" - //req, err = http.NewRequest("GET", url, nil) - //framework.ExpectNoError(err) - //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusOK)) - //s, err = getBody(resp) - //framework.ExpectNoError(err) - //Expect(s).To(Equal(backendContent)) - - // Test changed path - //newPath := "/foo" - //changePathIng := changePathIngress(updatedIng, newPath) - //ingressUpdate, err = cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), changePathIng, metav1.UpdateOptions{}) - //framework.ExpectNoError(err) - // - //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 404 for the old request, because of the path route", ingressUpdate.Namespace, ingressUpdate.Name)) - //resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusNotFound) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusNotFound)) - //pathURL := "https://" + hostName + newPath - //pathReq, err := http.NewRequest("GET", pathURL, nil) - //framework.ExpectNoError(err) - //By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 for a new request to the path route", ingressUpdate.Namespace, ingressUpdate.Name)) - //resp, err = getAndWaitResponse(rt, pathReq, 10*time.Second, http.StatusOK) - //framework.ExpectNoError(err) - //Expect(resp.StatusCode).To(Equal(http.StatusOK)) - //s, err = getBody(resp) - //framework.ExpectNoError(err) - //Expect(s).To(Equal(backendContent)) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) }) }) -var ___ = describe("Ingress tests paths", func() { +var ____ = describe("Ingress tests paths", func() { f := framework.NewDefaultFramework("skipper-ingress-paths") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -523,7 +615,7 @@ var ___ = describe("Ingress tests paths", func() { }) }) -var ____ = describe("Ingress tests custom routes", func() { +var _____ = describe("Ingress tests custom routes", func() { f := framework.NewDefaultFramework("skipper-ingress-custom") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -646,7 +738,7 @@ var ____ = describe("Ingress tests custom routes", func() { }) }) -var _____ = describe("Ingress tests paths", func() { +var ______ = describe("Ingress tests paths", func() { f := framework.NewDefaultFramework("skipper-ingress-paths") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -835,7 +927,7 @@ var _____ = describe("Ingress tests paths", func() { }) }) -var ______ = describe("Ingress tests custom routes", func() { +var _______ = describe("Ingress tests custom routes", func() { f := framework.NewDefaultFramework("skipper-ingress-custom") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -958,7 +1050,7 @@ var ______ = describe("Ingress tests custom routes", func() { }) }) -var _______ = describe("Ingress tests simple NLB", func() { +var ________ = describe("Ingress tests simple NLB", func() { f := framework.NewDefaultFramework("skipper-ingress-simple-nlb") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index bf75f441fc..eb8b2f5538 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[IngressOpa\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From aab71c036ea54720152b0f5337e829f9a0cabd89 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 16:24:21 +0100 Subject: [PATCH 11/44] Fix typo --- test/e2e/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index c216d00768..251c3b6ac6 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -323,7 +323,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) - It("Should activate OPA filter without issue [Ingress] [Zalando1]", func() { + It("Should activate OPA filter without issue [Ingress] [Zalando]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" From 97cd4d6722737838a29b06e06b7ac8b2fc46152a Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 6 Dec 2024 18:07:13 +0100 Subject: [PATCH 12/44] Try removing the valid token from the authz header --- test/e2e/ingress.go | 1 + 1 file changed, 1 insertion(+) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 251c3b6ac6..d8f8bbe769 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -435,6 +435,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { Expect(s).To(Equal(backendContent)) By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + req.Header.Del("Authorization") req.Header.Set("Authorization", "Basic invalid_token") //Unauthorized request resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) framework.ExpectNoError(err) From 5a063a4fbb69f4c7cec4e6e5311ae7c3710850cd Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Mon, 9 Dec 2024 08:37:08 +0100 Subject: [PATCH 13/44] Run single test. Forbidden first --- test/e2e/ingress.go | 18 ++++++++---------- test/e2e/run_e2e.sh | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index d8f8bbe769..e3a64dcd49 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -323,7 +323,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) - It("Should activate OPA filter without issue [Ingress] [Zalando]", func() { + It("Should activate OPA filter without issue [Ingress] [ZalandoIAM]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" @@ -417,15 +417,20 @@ var ___ = describe("Ingress tests for OPA filters", func() { netv1.PathTypeImplementationSpecific, ingressCreate.ObjectMeta.Labels, map[string]string{ - "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + "zalando.org/skipper-filter": `opaAuthorizeRequest("styra-smoketest")`, }, port, ) ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) framework.ExpectNoError(err) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(20 * time.Second) // wait for routing change propagation + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) - time.Sleep(10 * time.Second) // wait for routing change propagation req.Header.Set("Authorization", "Basic valid_token") //Authorized request resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) framework.ExpectNoError(err) @@ -433,13 +438,6 @@ var ___ = describe("Ingress tests for OPA filters", func() { s, err = getBody(resp) framework.ExpectNoError(err) Expect(s).To(Equal(backendContent)) - - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) - req.Header.Del("Authorization") - req.Header.Set("Authorization", "Basic invalid_token") //Unauthorized request - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) }) }) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index eb8b2f5538..1770ad0efc 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ + -focus="(\[ZalandoIAM\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From 3a7e603afa8c2ba2352e09b71e0b81b85f88fce3 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Mon, 9 Dec 2024 16:39:52 +0100 Subject: [PATCH 14/44] Update bucket policy to have the role --- delivery.yaml | 15 ++++++++++ test/e2e/apply/secret.yaml | 3 ++ test/e2e/cluster_config.sh | 6 ++-- test/e2e/run_e2e.sh | 7 +++++ test/e2e/update-opa-bundle-s3-policy.sh | 37 +++++++++++++++++++++++++ 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100755 test/e2e/update-opa-bundle-s3-policy.sh diff --git a/delivery.yaml b/delivery.yaml index 0d15788058..57a51db9ce 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -160,6 +160,21 @@ pipeline: secretKeyRef: name: kubernetes-e2e-config-secret key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" - name: CLUSTER_ADMIN_TOKEN valueFrom: secretKeyRef: diff --git a/test/e2e/apply/secret.yaml b/test/e2e/apply/secret.yaml index 37c5c13c95..83819035c5 100644 --- a/test/e2e/apply/secret.yaml +++ b/test/e2e/apply/secret.yaml @@ -20,3 +20,6 @@ data: OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN" SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ==" STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A==" + SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U=" + SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA==" + SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg==" diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index 72a6d110af..f397e58893 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -49,9 +49,9 @@ clusters: teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" skipper_open_policy_agent_enabled: "true" skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" - skipper_open_policy_agent_bucket_arn: "arn:aws:s3:::zalando-open-policy-agent-sandbox-bundles" - skipper_open_policy_agent_observability_url: "https://open-policy-agent-sandbox-observability.corporate-iam.zalan.do/v1" - skipper_open_policy_agent_bundles_url: "https://zalando-open-policy-agent-sandbox-bundles.s3.eu-central-1.amazonaws.com" + skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}" + skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}" + skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 1770ad0efc..88ee3dc2c5 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -159,6 +159,13 @@ if [ "$e2e" = true ]; then export S3_AWS_IAM_BUCKET="zalando-e2e-test-${AWS_ACCOUNT}-${LOCAL_ID}" export AWS_IAM_ROLE="${LOCAL_ID}-e2e-aws-iam-test" + + # update bucket policy to include new ingress role for bundle pulling + export SKIPPER_INGRESS_ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT}:role/${LOCAL_ID}-app-skipper-ingress" + export OPA_BUNDLES_BUCKET="zalando-open-policy-agent-sandbox-bundles" + + ./update-opa-bundle-s3-policy.sh $OPA_BUNDLES_BUCKET $SKIPPER_INGRESS_ROLE_ARN + # Run e2e tests # * conformance tests # * statefulset tests diff --git a/test/e2e/update-opa-bundle-s3-policy.sh b/test/e2e/update-opa-bundle-s3-policy.sh new file mode 100755 index 0000000000..ac3c33eafb --- /dev/null +++ b/test/e2e/update-opa-bundle-s3-policy.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -euo pipefail + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + exit 1 +fi + +BUCKET_NAME="$1" +INGRESS_ROLE_ARN="$2" +STATEMENT_ID="AllowS3ReadAccess" + +echo "Retrieving the current bucket policy for bucket: $BUCKET_NAME" +echo $(aws sts get-caller-identity) # TODO: remove + +POLICY=$(aws s3api get-bucket-policy --bucket "$BUCKET_NAME" --query "Policy" --output text) + +echo "Modifying the policy to include new ARN: $INGRESS_ROLE_ARN for statement Id: $STATEMENT_ID" +UPDATED_POLICY=$(echo "$POLICY" | jq --arg new_arn "$INGRESS_ROLE_ARN" --arg statement_id "$STATEMENT_ID" ' + .Statement |= map( + if .Sid == $statement_id then + if .Principal.AWS then + .Principal.AWS |= (if type == "string" then [.] + [$new_arn] else . + [$new_arn] end) + else + . + end + else + . + end + ) +') + +echo "Updating the bucket policy for bucket: $BUCKET_NAME" +aws s3api put-bucket-policy --bucket "$BUCKET_NAME" --policy "$UPDATED_POLICY" + +echo "Bucket policy updated successfully!" From 28ff0b3b11d79b9558bf76661561fc952c089f29 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Tue, 10 Dec 2024 17:02:06 +0100 Subject: [PATCH 15/44] Remove S3 update --- test/e2e/run_e2e.sh | 7 ----- test/e2e/update-opa-bundle-s3-policy.sh | 37 ------------------------- 2 files changed, 44 deletions(-) delete mode 100755 test/e2e/update-opa-bundle-s3-policy.sh diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 88ee3dc2c5..1770ad0efc 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -159,13 +159,6 @@ if [ "$e2e" = true ]; then export S3_AWS_IAM_BUCKET="zalando-e2e-test-${AWS_ACCOUNT}-${LOCAL_ID}" export AWS_IAM_ROLE="${LOCAL_ID}-e2e-aws-iam-test" - - # update bucket policy to include new ingress role for bundle pulling - export SKIPPER_INGRESS_ROLE_ARN="arn:aws:iam::${AWS_ACCOUNT}:role/${LOCAL_ID}-app-skipper-ingress" - export OPA_BUNDLES_BUCKET="zalando-open-policy-agent-sandbox-bundles" - - ./update-opa-bundle-s3-policy.sh $OPA_BUNDLES_BUCKET $SKIPPER_INGRESS_ROLE_ARN - # Run e2e tests # * conformance tests # * statefulset tests diff --git a/test/e2e/update-opa-bundle-s3-policy.sh b/test/e2e/update-opa-bundle-s3-policy.sh deleted file mode 100755 index ac3c33eafb..0000000000 --- a/test/e2e/update-opa-bundle-s3-policy.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [ "$#" -ne 2 ]; then - echo "Usage: $0 " - exit 1 -fi - -BUCKET_NAME="$1" -INGRESS_ROLE_ARN="$2" -STATEMENT_ID="AllowS3ReadAccess" - -echo "Retrieving the current bucket policy for bucket: $BUCKET_NAME" -echo $(aws sts get-caller-identity) # TODO: remove - -POLICY=$(aws s3api get-bucket-policy --bucket "$BUCKET_NAME" --query "Policy" --output text) - -echo "Modifying the policy to include new ARN: $INGRESS_ROLE_ARN for statement Id: $STATEMENT_ID" -UPDATED_POLICY=$(echo "$POLICY" | jq --arg new_arn "$INGRESS_ROLE_ARN" --arg statement_id "$STATEMENT_ID" ' - .Statement |= map( - if .Sid == $statement_id then - if .Principal.AWS then - .Principal.AWS |= (if type == "string" then [.] + [$new_arn] else . + [$new_arn] end) - else - . - end - else - . - end - ) -') - -echo "Updating the bucket policy for bucket: $BUCKET_NAME" -aws s3api put-bucket-policy --bucket "$BUCKET_NAME" --policy "$UPDATED_POLICY" - -echo "Bucket policy updated successfully!" From f33fed21b0ee0b7e376d190110c301fc45610bb5 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 12 Dec 2024 11:33:10 +0100 Subject: [PATCH 16/44] Use the opaPolicyName parameter --- test/e2e/ingress.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index e3a64dcd49..acf50db872 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -417,7 +417,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { netv1.PathTypeImplementationSpecific, ingressCreate.ObjectMeta.Labels, map[string]string{ - "zalando.org/skipper-filter": `opaAuthorizeRequest("styra-smoketest")`, + "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), }, port, ) From 24bddddd02af064395c9b312ae1fb2d801f6a6f4 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 13 Dec 2024 09:05:25 +0100 Subject: [PATCH 17/44] Add delay --- test/e2e/ingress.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index acf50db872..fcf5c1f609 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -323,6 +323,8 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) + time.Sleep(120 * time.Second) // see weather waiting helps with AWS_ROLE_ARN + It("Should activate OPA filter without issue [Ingress] [ZalandoIAM]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet From 5da6e216876d03973a9aa9bdbc4056558ebc8d29 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 13 Dec 2024 10:08:27 +0100 Subject: [PATCH 18/44] Remove delay --- test/e2e/ingress.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index fcf5c1f609..acf50db872 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -323,8 +323,6 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) - time.Sleep(120 * time.Second) // see weather waiting helps with AWS_ROLE_ARN - It("Should activate OPA filter without issue [Ingress] [ZalandoIAM]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet From 4b556c0e9a610efa70741b271e546f42a122aa5d Mon Sep 17 00:00:00 2001 From: Mikkel Oscar Lyderik Larsen Date: Fri, 13 Dec 2024 10:49:12 +0100 Subject: [PATCH 19/44] Order rbac before deployment Signed-off-by: Mikkel Oscar Lyderik Larsen --- cluster/manifests/skipper/{rbac.yaml => 01-rbac.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename cluster/manifests/skipper/{rbac.yaml => 01-rbac.yaml} (100%) diff --git a/cluster/manifests/skipper/rbac.yaml b/cluster/manifests/skipper/01-rbac.yaml similarity index 100% rename from cluster/manifests/skipper/rbac.yaml rename to cluster/manifests/skipper/01-rbac.yaml From 1d7e2018a7c7b29fc2fa5dfb6940a947f56fb0d9 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 13 Dec 2024 12:40:36 +0100 Subject: [PATCH 20/44] Run OPA filter tests as one of the [Zalando] tests --- test/e2e/ingress.go | 2 +- test/e2e/run_e2e.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index acf50db872..97b73a2b95 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -323,7 +323,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) - It("Should activate OPA filter without issue [Ingress] [ZalandoIAM]", func() { + It("Should activate OPA filter without issue [Ingress] [Zalando]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 1770ad0efc..eb8b2f5538 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[ZalandoIAM\])" \ + -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From d9e1e2d528f8cf154f4f9b962f0d150f62ca69bf Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 11:24:07 +0100 Subject: [PATCH 21/44] Separate opa testing --- delivery.yaml | 194 ++++++++++++++++++++++++++++++++++++++++++++ test/e2e/ingress.go | 5 +- test/e2e/run_e2e.sh | 20 ++++- 3 files changed, 212 insertions(+), 7 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 57a51db9ce..c82d64bdff 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -80,6 +80,8 @@ pipeline: value: "teapot-kubernetes-e2e-results" - name: ETCD_ENDPOINTS value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: SKIPPER_OPA_ENABLED + value: "false" - name: HOSTED_ZONE valueFrom: configMapKeyRef: @@ -262,6 +264,198 @@ pipeline: cpu: 1000m memory: 1000Mi +- id: create-opa-enabled-cluster + when: + event: pull_request + depends_on: + - build + type: process + desc: "Create cluster for OPA filter e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: &apply_env + - name: DEPLOYMENT_PATH + value: test/e2e + - name: IAM_ROLE_ARN + value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" + - name: APPLICATION + value: kubernetes + - name: COMPONENT + value: e2e + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - create-cluster + - true + env: &e2e_env + - name: CLUSTER_DOMAIN + value: teapot-e2e.zalan.do + - name: CLUSTER_DOMAIN_INTERNAL + value: ingress.cluster.local + - name: RESULT_BUCKET + value: "teapot-kubernetes-e2e-results" + - name: ETCD_ENDPOINTS + value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: SKIPPER_OPA_ENABLED + value: "true" + - name: HOSTED_ZONE + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "HOSTED_ZONE" + - name: REGION + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "REGION" + - name: AWS_ACCOUNT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AWS_ACCOUNT" + - name: ZMON_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ZMON_ROOT_ACCOUNT_ROLE" + - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" + - name: SESSION_MANAGER_DESTINATION_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SESSION_MANAGER_DESTINATION_ARN" + - name: APISERVER_BUSINESS_PARTNER_IDS + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "APISERVER_BUSINESS_PARTNER_IDS" + - name: LIGHTSTEP_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "LIGHTSTEP_TOKEN" + - name: OWNER + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OWNER" + - name: VPC_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "VPC_ID" + - name: EFS_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "EFS_ID" + - name: ETCD_CLIENT_CA_CERT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_CERT" + - name: ETCD_CLIENT_CA_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_KEY" + - name: ETCD_SCALYR_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_SCALYR_KEY" + - name: OKTA_AUTH_ISSUER_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" + - name: CLUSTER_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-credentials + key: "cluster-token-secret" + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi +- id: opa-e2e-tests + when: + event: pull_request + depends_on: + - create-opa-enabled-cluster + type: process + desc: "OPA filter in Kubernetes e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - e2e + - true + env: *e2e_env + resources: + limits: + cpu: 2 + memory: 8Gi + requests: + cpu: 2 + memory: 8Gi - id: stackset-e2e-tests when: event: pull_request diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 97b73a2b95..2119dcd431 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -19,9 +19,6 @@ import ( "net/http" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -323,7 +320,7 @@ var ___ = describe("Ingress tests for OPA filters", func() { jig *ingress.TestJig ) - It("Should activate OPA filter without issue [Ingress] [Zalando]", func() { + It("Should activate OPA filter without issue [Ingress] [Opa]", func() { jig = ingress.NewIngressTestJig(f.ClientSet) cs = f.ClientSet serviceName := "skipper-ingress-test" diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index eb8b2f5538..daff3973dc 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -7,6 +7,7 @@ loadtest_e2e=false stackset_e2e=false decommission_cluster=false COMMAND="${1:-"all"}" # all, create-cluster, e2e, stackset-e2e, decommission-cluster +OPA_ENABLED="${2:-false}" case "$COMMAND" in all) @@ -45,8 +46,15 @@ CDP_TARGET_COMMIT_ID="${CDP_TARGET_COMMIT_ID:-"dev"}" CDP_HEAD_COMMIT_ID="${CDP_HEAD_COMMIT_ID:-"$(git describe --tags --always)"}" RESULT_BUCKET="${RESULT_BUCKET:-""}" -export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" -export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" +if [ "$OPA_ENABLED" == "true" ]; then + echo "Running with OPA specific configuration" + export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e_opa-${CDP_BUILD_VERSION}"}" + export LOCAL_ID="${LOCAL_ID:-"e2e_opa-${CDP_BUILD_VERSION}"}" +else + export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" + export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" +fi + export API_SERVER_URL="https://${LOCAL_ID}.${HOSTED_ZONE}" export INFRASTRUCTURE_ACCOUNT="aws:${AWS_ACCOUNT}" export CLUSTER_ID="${INFRASTRUCTURE_ACCOUNT}:${REGION}:${LOCAL_ID}" @@ -181,9 +189,15 @@ if [ "$e2e" = true ]; then # introduce a broken DNS record to mess with ExternalDNS # kubectl apply -f broken-dns-record.yaml + if [ "$OPA_ENABLED" == "true" ]; then + FOCUS="(\[Opa\])" + else + FOCUS="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" + fi + mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ + -focus="$FOCUS" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \ From be8d409a7500e8ffaef5c31d0fa41121c19287cb Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 11:26:57 +0100 Subject: [PATCH 22/44] Separate opa testing --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index c82d64bdff..f1b391de23 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -275,7 +275,7 @@ pipeline: process: microservice_standard_test config: apply_manifests: - env: &apply_env + env: &apply_opa_enabled_env - name: DEPLOYMENT_PATH value: test/e2e - name: IAM_ROLE_ARN @@ -430,7 +430,7 @@ pipeline: process: microservice_standard_test config: apply_manifests: - env: *apply_env + env: *apply_opa_enabled_env end2end_tests: metadata: name: e2e From 584eda07ebd45ef0445c4d0556cd8e58989be1a0 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 11:28:23 +0100 Subject: [PATCH 23/44] Fix var conflict --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index f1b391de23..1fc15ad567 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -301,7 +301,7 @@ pipeline: args: - create-cluster - true - env: &e2e_env + env: &opa_e2e_env - name: CLUSTER_DOMAIN value: teapot-e2e.zalan.do - name: CLUSTER_DOMAIN_INTERNAL @@ -448,7 +448,7 @@ pipeline: args: - e2e - true - env: *e2e_env + env: *opa_e2e_env resources: limits: cpu: 2 From 3390c7e35419e9f374754092a3f3984df5d2d512 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 12:43:02 +0100 Subject: [PATCH 24/44] Add missing deps --- test/e2e/ingress.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 2119dcd431..c0f473ea38 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -19,6 +19,9 @@ import ( "net/http" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" From 5751d72a626259c1e58f7f3a2bf208c34c034631 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 12:46:04 +0100 Subject: [PATCH 25/44] Use env variable --- test/e2e/cluster_config.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index f397e58893..82e881bd4a 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -47,7 +47,7 @@ clusters: karpenter_pools_enabled: "true" okta_auth_client_id: "kubernetes.cluster.teapot-e2e" teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" - skipper_open_policy_agent_enabled: "true" + skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}" skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}" skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}" From cc0e631804e2ed8307a96fa31391fe81a4bd032e Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 13:03:59 +0100 Subject: [PATCH 26/44] Quote boolean in yaml --- delivery.yaml | 4 +- test/e2e/ingress.go | 265 ++++++++++++++++++++++---------------------- 2 files changed, 133 insertions(+), 136 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 1fc15ad567..f130e9e96c 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -300,7 +300,7 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - create-cluster - - true + - "true" env: &opa_e2e_env - name: CLUSTER_DOMAIN value: teapot-e2e.zalan.do @@ -447,7 +447,7 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - e2e - - true + - "true" env: *opa_e2e_env resources: limits: diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index c0f473ea38..650c9670c0 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -19,9 +19,6 @@ import ( "net/http" "time" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -315,133 +312,7 @@ var __ = describe("Ingress tests simple", func() { }) }) -var ___ = describe("Ingress tests for OPA filters", func() { - f := framework.NewDefaultFramework("skipper-ingress-with-opa") - f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline - var ( - cs kubernetes.Interface - jig *ingress.TestJig - ) - - It("Should activate OPA filter without issue [Ingress] [Opa]", func() { - jig = ingress.NewIngressTestJig(f.ClientSet) - cs = f.ClientSet - serviceName := "skipper-ingress-test" - ns := f.Namespace.Name - hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone()) - labels := map[string]string{ - "app": serviceName, - } - port := 8080 - replicas := int32(3) - targetPort := 9090 - backendContent := "mytest" - route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) - waitTime := 10 * time.Minute - - // CREATE setup - // backend deployment - By("Creating a deployment with " + serviceName + " in namespace " + ns) - depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) - _, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - By("Creating service " + serviceName + " in namespace " + ns) - service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) - _, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port) - ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) - framework.ExpectNoError(err) - - addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime) - framework.ExpectNoError(err) - - _, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"}) - framework.ExpectNoError(err) - - // skipper http -> https redirect - By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") - err = waitForResponse(addr, "http", waitTime, isRedirect, true) - framework.ExpectNoError(err) - - // ALB ready - By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") - err = waitForResponse(addr, "https", waitTime, isNotFound, true) - framework.ExpectNoError(err) - - // DNS ready - By("Waiting for DNS to see that external-dns and skipper route to service and pod works") - err = waitForResponse(hostName, "https", waitTime, isSuccess, false) - framework.ExpectNoError(err) - - // Test that we get content from the default ingress - By("By checking the content of the reply we see that the ingress stack works") - rt, quit := createHTTPRoundTripper() - defer func() { - quit <- struct{}{} - }() - url := "https://" + hostName + "/" - req, err := http.NewRequest("GET", url, nil) - framework.ExpectNoError(err) - resp, err := rt.RoundTrip(req) - framework.ExpectNoError(err) - s, err := getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) - - // Start actual ingress tests - // Test ingress Filters: opaAuthorizeRequest - - /** - ## The Rule looks like below. - ## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego - default allow := false - - allow if { - input.attributes.request.http.method == "GET" - auth_header_val := input.attributes.request.http.headers.authorization - startswith(auth_header_val, "Basic ") - token := substring(auth_header_val, count("Basic "), -1) - token == "valid_token" - } - */ - path := "/" - opaPolicyName := "styra-smoketest" - updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, - ingressCreate.ObjectMeta.Namespace, - hostName, - serviceName, - path, - netv1.PathTypeImplementationSpecific, - ingressCreate.ObjectMeta.Labels, - map[string]string{ - "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), - }, - port, - ) - ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) - framework.ExpectNoError(err) - - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) - time.Sleep(20 * time.Second) // wait for routing change propagation - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) - - By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) - req.Header.Set("Authorization", "Basic valid_token") //Authorized request - resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) - framework.ExpectNoError(err) - Expect(resp.StatusCode).To(Equal(http.StatusOK)) - s, err = getBody(resp) - framework.ExpectNoError(err) - Expect(s).To(Equal(backendContent)) - }) -}) - -var ____ = describe("Ingress tests paths", func() { +var ___ = describe("Ingress tests paths", func() { f := framework.NewDefaultFramework("skipper-ingress-paths") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -614,7 +485,7 @@ var ____ = describe("Ingress tests paths", func() { }) }) -var _____ = describe("Ingress tests custom routes", func() { +var ____ = describe("Ingress tests custom routes", func() { f := framework.NewDefaultFramework("skipper-ingress-custom") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -737,7 +608,7 @@ var _____ = describe("Ingress tests custom routes", func() { }) }) -var ______ = describe("Ingress tests paths", func() { +var _____ = describe("Ingress tests paths", func() { f := framework.NewDefaultFramework("skipper-ingress-paths") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -926,7 +797,7 @@ var ______ = describe("Ingress tests paths", func() { }) }) -var _______ = describe("Ingress tests custom routes", func() { +var ______ = describe("Ingress tests custom routes", func() { f := framework.NewDefaultFramework("skipper-ingress-custom") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -1049,7 +920,7 @@ var _______ = describe("Ingress tests custom routes", func() { }) }) -var ________ = describe("Ingress tests simple NLB", func() { +var _______ = describe("Ingress tests simple NLB", func() { f := framework.NewDefaultFramework("skipper-ingress-simple-nlb") f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline var ( @@ -1153,3 +1024,129 @@ var ________ = describe("Ingress tests simple NLB", func() { Expect(resp.Header.Get("Request-Host")).To(Equal(hostName)) }) }) + +var ________ = describe("Ingress tests for OPA filters", func() { + f := framework.NewDefaultFramework("skipper-ingress-with-opa") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + var ( + cs kubernetes.Interface + jig *ingress.TestJig + ) + + It("Should activate OPA filter without issue [Ingress] [Opa]", func() { + jig = ingress.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "mytest" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + waitTime := 10 * time.Minute + + // CREATE setup + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + _, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port) + ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime) + framework.ExpectNoError(err) + + _, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + framework.ExpectNoError(err) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + framework.ExpectNoError(err) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + framework.ExpectNoError(err) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + framework.ExpectNoError(err) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + framework.ExpectNoError(err) + resp, err := rt.RoundTrip(req) + framework.ExpectNoError(err) + s, err := getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + + // Start actual ingress tests + // Test ingress Filters: opaAuthorizeRequest + + /** + ## The Rule looks like below. + ## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego + default allow := false + + allow if { + input.attributes.request.http.method == "GET" + auth_header_val := input.attributes.request.http.headers.authorization + startswith(auth_header_val, "Basic ") + token := substring(auth_header_val, count("Basic "), -1) + token == "valid_token" + } + */ + path := "/" + opaPolicyName := "styra-smoketest" + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + netv1.PathTypeImplementationSpecific, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + }, + port, + ) + ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(20 * time.Second) // wait for routing change propagation + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + req.Header.Set("Authorization", "Basic valid_token") //Authorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + }) +}) From 4f5dda350e67fd0ae8603a6e5a66ba0ca9bba75b Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 13:05:16 +0100 Subject: [PATCH 27/44] Add missing deps --- test/e2e/ingress.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 650c9670c0..a80189c5c4 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -19,6 +19,9 @@ import ( "net/http" "time" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" From 10b6d59006c0c9fde252ef47cfb91ec8c0c615e3 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 13:17:52 +0100 Subject: [PATCH 28/44] Adhere to the pattern: [a-zA-Z] --- test/e2e/run_e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index daff3973dc..bbb2d47535 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -48,8 +48,8 @@ RESULT_BUCKET="${RESULT_BUCKET:-""}" if [ "$OPA_ENABLED" == "true" ]; then echo "Running with OPA specific configuration" - export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e_opa-${CDP_BUILD_VERSION}"}" - export LOCAL_ID="${LOCAL_ID:-"e2e_opa-${CDP_BUILD_VERSION}"}" + export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2eopa-${CDP_BUILD_VERSION}"}" + export LOCAL_ID="${LOCAL_ID:-"e2eopa-${CDP_BUILD_VERSION}"}" else export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" From 41667dee311ae04516232138ae3d7ab754217776 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 14:04:30 +0100 Subject: [PATCH 29/44] Try higher timeout --- test/e2e/run_e2e.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index bbb2d47535..752935a9f2 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -122,7 +122,7 @@ if [ "$create_cluster" = true ]; then --manage-etcd-stack # Wait for the resources to be ready - ./wait-for-update.py --timeout 1200 + ./wait-for-update.py --timeout 1500 # provision and start load test echo "provision and start load test" From 38f1e79af6df3f6404a4b497fe47fbdd9ce9d049 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 15:30:59 +0100 Subject: [PATCH 30/44] Skip general cluster creation --- delivery.yaml | 456 +++++++++++++++++++++++++------------------------- 1 file changed, 228 insertions(+), 228 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index f130e9e96c..3958b89c7f 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -35,234 +35,234 @@ pipeline: IMAGE=container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e IMAGE=$IMAGE VERSION=$CDP_BUILD_VERSION make build.push.multiarch -- id: create-cluster - when: - event: pull_request - depends_on: - - build - type: process - desc: "Create cluster for Kubernetes e2e tests" - target: stups-test - process: microservice_standard_test - config: - apply_manifests: - env: &apply_env - - name: DEPLOYMENT_PATH - value: test/e2e - - name: IAM_ROLE_ARN - value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" - - name: APPLICATION - value: kubernetes - - name: COMPONENT - value: e2e - end2end_tests: - metadata: - name: e2e - labels: - application: kubernetes - component: e2e - annotations: - zalando.org/runtime-policy: require-on-demand - spec: - serviceAccountName: kubernetes-e2e - restartPolicy: Never - containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - create-cluster - env: &e2e_env - - name: CLUSTER_DOMAIN - value: teapot-e2e.zalan.do - - name: CLUSTER_DOMAIN_INTERNAL - value: ingress.cluster.local - - name: RESULT_BUCKET - value: "teapot-kubernetes-e2e-results" - - name: ETCD_ENDPOINTS - value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" - - name: SKIPPER_OPA_ENABLED - value: "false" - - name: HOSTED_ZONE - valueFrom: - configMapKeyRef: - name: kubernetes-e2e-config - key: "HOSTED_ZONE" - - name: REGION - valueFrom: - configMapKeyRef: - name: kubernetes-e2e-config - key: "REGION" - - name: AWS_ACCOUNT - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "AWS_ACCOUNT" - - name: ZMON_ROOT_ACCOUNT_ROLE - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "ZMON_ROOT_ACCOUNT_ROLE" - - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" - - name: SESSION_MANAGER_DESTINATION_ARN - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "SESSION_MANAGER_DESTINATION_ARN" - - name: APISERVER_BUSINESS_PARTNER_IDS - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "APISERVER_BUSINESS_PARTNER_IDS" - - name: LIGHTSTEP_TOKEN - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "LIGHTSTEP_TOKEN" - - name: OWNER - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "OWNER" - - name: VPC_ID - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "VPC_ID" - - name: EFS_ID - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "EFS_ID" - - name: ETCD_CLIENT_CA_CERT - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "ETCD_CLIENT_CA_CERT" - - name: ETCD_CLIENT_CA_KEY - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "ETCD_CLIENT_CA_KEY" - - name: ETCD_SCALYR_KEY - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "ETCD_SCALYR_KEY" - - name: OKTA_AUTH_ISSUER_URL - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "OKTA_AUTH_ISSUER_URL" - - name: STYRA_TOKEN - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "STYRA_TOKEN" - - name: SKIPPER_OPA_BUCKET_ARN - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "SKIPPER_OPA_BUCKET_ARN" - - name: SKIPPER_OPA_OBSERVABILITY_URL - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "SKIPPER_OPA_OBSERVABILITY_URL" - - name: SKIPPER_OPA_BUNDLES_URL - valueFrom: - secretKeyRef: - name: kubernetes-e2e-config-secret - key: "SKIPPER_OPA_BUNDLES_URL" - - name: CLUSTER_ADMIN_TOKEN - valueFrom: - secretKeyRef: - name: kubernetes-e2e-credentials - key: "cluster-token-secret" - resources: - limits: - cpu: 500m - memory: 2Gi - requests: - cpu: 500m - memory: 2Gi - -- id: e2e-tests - when: - event: pull_request - depends_on: - - create-cluster - type: process - desc: "Kubernetes e2e tests" - target: stups-test - process: microservice_standard_test - config: - apply_manifests: - env: *apply_env - end2end_tests: - metadata: - name: e2e - labels: - application: kubernetes - component: e2e - annotations: - zalando.org/runtime-policy: require-on-demand - spec: - serviceAccountName: kubernetes-e2e - restartPolicy: Never - containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - e2e - env: *e2e_env - resources: - limits: - cpu: 2 - memory: 8Gi - requests: - cpu: 2 - memory: 8Gi - -- id: e2e-load-test-result - when: - event: pull_request - depends_on: - - create-cluster - type: process - desc: "Kubernetes e2e load tests" - target: stups-test - process: microservice_standard_test - config: - apply_manifests: - env: *apply_env - end2end_tests: - metadata: - name: e2e - labels: - application: kubernetes - component: e2e - annotations: - zalando.org/runtime-policy: require-on-demand - spec: - serviceAccountName: kubernetes-e2e - restartPolicy: Never - containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - loadtest-e2e - env: *e2e_env - resources: - limits: - cpu: 1000m - memory: 1000Mi - requests: - cpu: 1000m - memory: 1000Mi +#- id: create-cluster +# when: +# event: pull_request +# depends_on: +# - build +# type: process +# desc: "Create cluster for Kubernetes e2e tests" +# target: stups-test +# process: microservice_standard_test +# config: +# apply_manifests: +# env: &apply_env +# - name: DEPLOYMENT_PATH +# value: test/e2e +# - name: IAM_ROLE_ARN +# value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" +# - name: APPLICATION +# value: kubernetes +# - name: COMPONENT +# value: e2e +# end2end_tests: +# metadata: +# name: e2e +# labels: +# application: kubernetes +# component: e2e +# annotations: +# zalando.org/runtime-policy: require-on-demand +# spec: +# serviceAccountName: kubernetes-e2e +# restartPolicy: Never +# containers: +# - name: e2e +# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" +# args: +# - create-cluster +# env: &e2e_env +# - name: CLUSTER_DOMAIN +# value: teapot-e2e.zalan.do +# - name: CLUSTER_DOMAIN_INTERNAL +# value: ingress.cluster.local +# - name: RESULT_BUCKET +# value: "teapot-kubernetes-e2e-results" +# - name: ETCD_ENDPOINTS +# value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" +# - name: SKIPPER_OPA_ENABLED +# value: "false" +# - name: HOSTED_ZONE +# valueFrom: +# configMapKeyRef: +# name: kubernetes-e2e-config +# key: "HOSTED_ZONE" +# - name: REGION +# valueFrom: +# configMapKeyRef: +# name: kubernetes-e2e-config +# key: "REGION" +# - name: AWS_ACCOUNT +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "AWS_ACCOUNT" +# - name: ZMON_ROOT_ACCOUNT_ROLE +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "ZMON_ROOT_ACCOUNT_ROLE" +# - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" +# - name: SESSION_MANAGER_DESTINATION_ARN +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "SESSION_MANAGER_DESTINATION_ARN" +# - name: APISERVER_BUSINESS_PARTNER_IDS +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "APISERVER_BUSINESS_PARTNER_IDS" +# - name: LIGHTSTEP_TOKEN +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "LIGHTSTEP_TOKEN" +# - name: OWNER +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "OWNER" +# - name: VPC_ID +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "VPC_ID" +# - name: EFS_ID +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "EFS_ID" +# - name: ETCD_CLIENT_CA_CERT +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "ETCD_CLIENT_CA_CERT" +# - name: ETCD_CLIENT_CA_KEY +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "ETCD_CLIENT_CA_KEY" +# - name: ETCD_SCALYR_KEY +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "ETCD_SCALYR_KEY" +# - name: OKTA_AUTH_ISSUER_URL +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "OKTA_AUTH_ISSUER_URL" +# - name: STYRA_TOKEN +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "STYRA_TOKEN" +# - name: SKIPPER_OPA_BUCKET_ARN +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "SKIPPER_OPA_BUCKET_ARN" +# - name: SKIPPER_OPA_OBSERVABILITY_URL +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "SKIPPER_OPA_OBSERVABILITY_URL" +# - name: SKIPPER_OPA_BUNDLES_URL +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-config-secret +# key: "SKIPPER_OPA_BUNDLES_URL" +# - name: CLUSTER_ADMIN_TOKEN +# valueFrom: +# secretKeyRef: +# name: kubernetes-e2e-credentials +# key: "cluster-token-secret" +# resources: +# limits: +# cpu: 500m +# memory: 2Gi +# requests: +# cpu: 500m +# memory: 2Gi +# +#- id: e2e-tests +# when: +# event: pull_request +# depends_on: +# - create-cluster +# type: process +# desc: "Kubernetes e2e tests" +# target: stups-test +# process: microservice_standard_test +# config: +# apply_manifests: +# env: *apply_env +# end2end_tests: +# metadata: +# name: e2e +# labels: +# application: kubernetes +# component: e2e +# annotations: +# zalando.org/runtime-policy: require-on-demand +# spec: +# serviceAccountName: kubernetes-e2e +# restartPolicy: Never +# containers: +# - name: e2e +# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" +# args: +# - e2e +# env: *e2e_env +# resources: +# limits: +# cpu: 2 +# memory: 8Gi +# requests: +# cpu: 2 +# memory: 8Gi +# +#- id: e2e-load-test-result +# when: +# event: pull_request +# depends_on: +# - create-cluster +# type: process +# desc: "Kubernetes e2e load tests" +# target: stups-test +# process: microservice_standard_test +# config: +# apply_manifests: +# env: *apply_env +# end2end_tests: +# metadata: +# name: e2e +# labels: +# application: kubernetes +# component: e2e +# annotations: +# zalando.org/runtime-policy: require-on-demand +# spec: +# serviceAccountName: kubernetes-e2e +# restartPolicy: Never +# containers: +# - name: e2e +# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" +# args: +# - loadtest-e2e +# env: *e2e_env +# resources: +# limits: +# cpu: 1000m +# memory: 1000Mi +# requests: +# cpu: 1000m +# memory: 1000Mi - id: create-opa-enabled-cluster when: From 982ad21716fb9d68a409b42b51319ac3ef311a84 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 15:31:58 +0100 Subject: [PATCH 31/44] Skip general cluster creation --- delivery.yaml | 72 +++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 3958b89c7f..aa3cac243f 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -456,42 +456,42 @@ pipeline: requests: cpu: 2 memory: 8Gi -- id: stackset-e2e-tests - when: - event: pull_request - depends_on: - - create-cluster - type: process - desc: "StackSet e2e tests" - target: stups-test - process: microservice_standard_test - config: - apply_manifests: - env: *apply_env - end2end_tests: - metadata: - name: e2e - labels: - application: kubernetes - component: e2e - annotations: - zalando.org/runtime-policy: require-on-demand - spec: - serviceAccountName: kubernetes-e2e - restartPolicy: Never - containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - stackset-e2e - env: *e2e_env - resources: - limits: - cpu: 2 - memory: 1Gi - requests: - cpu: 2 - memory: 1Gi +#- id: stackset-e2e-tests +# when: +# event: pull_request +# depends_on: +# - create-cluster +# type: process +# desc: "StackSet e2e tests" +# target: stups-test +# process: microservice_standard_test +# config: +# apply_manifests: +# env: *apply_env +# end2end_tests: +# metadata: +# name: e2e +# labels: +# application: kubernetes +# component: e2e +# annotations: +# zalando.org/runtime-policy: require-on-demand +# spec: +# serviceAccountName: kubernetes-e2e +# restartPolicy: Never +# containers: +# - name: e2e +# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" +# args: +# - stackset-e2e +# env: *e2e_env +# resources: +# limits: +# cpu: 2 +# memory: 1Gi +# requests: +# cpu: 2 +# memory: 1Gi - id: decommission-cluster when: From 4207939d53a98e8d1261ea967f0b3deeb4e0219e Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 15:33:06 +0100 Subject: [PATCH 32/44] Skip general cluster creation --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index aa3cac243f..f9cd040204 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -523,7 +523,7 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - decommission-cluster - env: *e2e_env + env: *apply_opa_enabled_env resources: limits: cpu: 500m From 611b9c18c0b16f295fcddf38ef18b14c5977e386 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 15:37:28 +0100 Subject: [PATCH 33/44] Skip general cluster creation --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index f9cd040204..e8077f27bb 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -506,7 +506,7 @@ pipeline: process: microservice_standard_test config: apply_manifests: - env: *apply_env + env: *apply_opa_enabled_env end2end_tests: metadata: name: e2e From dd646b9c5eddb7ec6e936e5ba0b52356323af007 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 15:38:43 +0100 Subject: [PATCH 34/44] Change depends on --- delivery.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index e8077f27bb..8680fb3ebe 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -497,9 +497,7 @@ pipeline: when: event: pull_request depends_on: - - e2e-tests - - stackset-e2e-tests - - e2e-load-test-result + - opa-e2e-tests type: process desc: "Decommission cluster used for Kubernetes e2e tests" target: stups-test From b66115d53eb451a58c339c862290937cb995daf7 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 16:48:05 +0100 Subject: [PATCH 35/44] Change component name --- delivery.yaml | 14 +++++++------- test/e2e/run_e2e.sh | 8 +++++++- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 8680fb3ebe..92311a719c 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -283,20 +283,20 @@ pipeline: - name: APPLICATION value: kubernetes - name: COMPONENT - value: e2e + value: e2eopa end2end_tests: metadata: - name: e2e + name: e2eopa labels: application: kubernetes - component: e2e + component: e2eopa annotations: zalando.org/runtime-policy: require-on-demand spec: serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2e + - name: e2eopa image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - create-cluster @@ -433,17 +433,17 @@ pipeline: env: *apply_opa_enabled_env end2end_tests: metadata: - name: e2e + name: e2eopa labels: application: kubernetes - component: e2e + component: e2eopa annotations: zalando.org/runtime-policy: require-on-demand spec: serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2e + - name: e2eopa image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - e2e diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 752935a9f2..af36eaf91f 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -102,6 +102,12 @@ if [ "$create_cluster" = true ]; then git clone "https://$CDP_TARGET_REPOSITORY" "$BASE_CFG_PATH" git -C "$BASE_CFG_PATH" reset --hard "${CDP_TARGET_COMMIT_ID}" +# if [ "$OPA_ENABLED" == "true" ]; then +# echo "Removing some cluster features to speed up the creation" +# rm -rf "$BASE_CFG_PATH/cluster/manifests/skipper-canary-controller" +# rm -rf "$BASE_CFG_PATH/cluster/manifests/prometheus" +# fi + # generate cluster.yaml # call the cluster_config.sh from base git checkout if possible if [ -f "$BASE_CFG_PATH/test/e2e/cluster_config.sh" ]; then @@ -122,7 +128,7 @@ if [ "$create_cluster" = true ]; then --manage-etcd-stack # Wait for the resources to be ready - ./wait-for-update.py --timeout 1500 + ./wait-for-update.py --timeout 1200 # provision and start load test echo "provision and start load test" From ebfacf29fe0fcbcc47eff7665cce431ed302b0ae Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 16:58:37 +0100 Subject: [PATCH 36/44] Change service account name --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index 92311a719c..e9d7347f45 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -293,7 +293,7 @@ pipeline: annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2e + serviceAccountName: kubernetes-e2eopa restartPolicy: Never containers: - name: e2eopa From 91447e435489c0271d0e8a94ee9f6479787736f4 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Thu, 19 Dec 2024 16:59:38 +0100 Subject: [PATCH 37/44] Change service account name --- delivery.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index e9d7347f45..e4bee40b36 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -440,7 +440,7 @@ pipeline: annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2e + serviceAccountName: kubernetes-e2eopa restartPolicy: Never containers: - name: e2eopa @@ -514,7 +514,7 @@ pipeline: annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2e + serviceAccountName: kubernetes-e2eopa restartPolicy: Never containers: - name: e2e From 333a050256175eb54785b5489d64563c8da0159d Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 20 Dec 2024 10:35:44 +0100 Subject: [PATCH 38/44] Use the name e2e --- delivery.yaml | 20 ++++++++++---------- test/e2e/run_e2e.sh | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index e4bee40b36..8680fb3ebe 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -283,20 +283,20 @@ pipeline: - name: APPLICATION value: kubernetes - name: COMPONENT - value: e2eopa + value: e2e end2end_tests: metadata: - name: e2eopa + name: e2e labels: application: kubernetes - component: e2eopa + component: e2e annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2eopa + serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2eopa + - name: e2e image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - create-cluster @@ -433,17 +433,17 @@ pipeline: env: *apply_opa_enabled_env end2end_tests: metadata: - name: e2eopa + name: e2e labels: application: kubernetes - component: e2eopa + component: e2e annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2eopa + serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2eopa + - name: e2e image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - e2e @@ -514,7 +514,7 @@ pipeline: annotations: zalando.org/runtime-policy: require-on-demand spec: - serviceAccountName: kubernetes-e2eopa + serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - name: e2e diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index af36eaf91f..0d722c14c7 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -48,8 +48,8 @@ RESULT_BUCKET="${RESULT_BUCKET:-""}" if [ "$OPA_ENABLED" == "true" ]; then echo "Running with OPA specific configuration" - export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2eopa-${CDP_BUILD_VERSION}"}" - export LOCAL_ID="${LOCAL_ID:-"e2eopa-${CDP_BUILD_VERSION}"}" + export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" + export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" else export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" From 30ef2cbdffc538ab633b4ad9e64d706e1b59b050 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 20 Dec 2024 11:34:31 +0100 Subject: [PATCH 39/44] Fix variable set --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index 8680fb3ebe..a1da714bfb 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -521,7 +521,7 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - decommission-cluster - env: *apply_opa_enabled_env + env: *opa_e2e_env resources: limits: cpu: 500m From b656b78b40914063afc205fac825611eb16e0b34 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 20 Dec 2024 12:15:58 +0100 Subject: [PATCH 40/44] Change local id and alias --- test/e2e/run_e2e.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index 0d722c14c7..df5892c4eb 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -48,8 +48,8 @@ RESULT_BUCKET="${RESULT_BUCKET:-""}" if [ "$OPA_ENABLED" == "true" ]; then echo "Running with OPA specific configuration" - export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" - export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" + export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}opa"}" + export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}opa"}" else export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" From 47e591acba5c6fdebe723dc1357b2da151b35216 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 20 Dec 2024 13:02:04 +0100 Subject: [PATCH 41/44] Uncomment old steps --- delivery.yaml | 614 ++++++++++++++++++++++++++------------------------ 1 file changed, 324 insertions(+), 290 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index a1da714bfb..77feeebeb6 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -35,234 +35,310 @@ pipeline: IMAGE=container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e IMAGE=$IMAGE VERSION=$CDP_BUILD_VERSION make build.push.multiarch -#- id: create-cluster -# when: -# event: pull_request -# depends_on: -# - build -# type: process -# desc: "Create cluster for Kubernetes e2e tests" -# target: stups-test -# process: microservice_standard_test -# config: -# apply_manifests: -# env: &apply_env -# - name: DEPLOYMENT_PATH -# value: test/e2e -# - name: IAM_ROLE_ARN -# value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" -# - name: APPLICATION -# value: kubernetes -# - name: COMPONENT -# value: e2e -# end2end_tests: -# metadata: -# name: e2e -# labels: -# application: kubernetes -# component: e2e -# annotations: -# zalando.org/runtime-policy: require-on-demand -# spec: -# serviceAccountName: kubernetes-e2e -# restartPolicy: Never -# containers: -# - name: e2e -# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" -# args: -# - create-cluster -# env: &e2e_env -# - name: CLUSTER_DOMAIN -# value: teapot-e2e.zalan.do -# - name: CLUSTER_DOMAIN_INTERNAL -# value: ingress.cluster.local -# - name: RESULT_BUCKET -# value: "teapot-kubernetes-e2e-results" -# - name: ETCD_ENDPOINTS -# value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" -# - name: SKIPPER_OPA_ENABLED -# value: "false" -# - name: HOSTED_ZONE -# valueFrom: -# configMapKeyRef: -# name: kubernetes-e2e-config -# key: "HOSTED_ZONE" -# - name: REGION -# valueFrom: -# configMapKeyRef: -# name: kubernetes-e2e-config -# key: "REGION" -# - name: AWS_ACCOUNT -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "AWS_ACCOUNT" -# - name: ZMON_ROOT_ACCOUNT_ROLE -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "ZMON_ROOT_ACCOUNT_ROLE" -# - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" -# - name: SESSION_MANAGER_DESTINATION_ARN -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "SESSION_MANAGER_DESTINATION_ARN" -# - name: APISERVER_BUSINESS_PARTNER_IDS -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "APISERVER_BUSINESS_PARTNER_IDS" -# - name: LIGHTSTEP_TOKEN -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "LIGHTSTEP_TOKEN" -# - name: OWNER -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "OWNER" -# - name: VPC_ID -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "VPC_ID" -# - name: EFS_ID -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "EFS_ID" -# - name: ETCD_CLIENT_CA_CERT -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "ETCD_CLIENT_CA_CERT" -# - name: ETCD_CLIENT_CA_KEY -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "ETCD_CLIENT_CA_KEY" -# - name: ETCD_SCALYR_KEY -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "ETCD_SCALYR_KEY" -# - name: OKTA_AUTH_ISSUER_URL -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "OKTA_AUTH_ISSUER_URL" -# - name: STYRA_TOKEN -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "STYRA_TOKEN" -# - name: SKIPPER_OPA_BUCKET_ARN -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "SKIPPER_OPA_BUCKET_ARN" -# - name: SKIPPER_OPA_OBSERVABILITY_URL -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "SKIPPER_OPA_OBSERVABILITY_URL" -# - name: SKIPPER_OPA_BUNDLES_URL -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-config-secret -# key: "SKIPPER_OPA_BUNDLES_URL" -# - name: CLUSTER_ADMIN_TOKEN -# valueFrom: -# secretKeyRef: -# name: kubernetes-e2e-credentials -# key: "cluster-token-secret" -# resources: -# limits: -# cpu: 500m -# memory: 2Gi -# requests: -# cpu: 500m -# memory: 2Gi -# -#- id: e2e-tests -# when: -# event: pull_request -# depends_on: -# - create-cluster -# type: process -# desc: "Kubernetes e2e tests" -# target: stups-test -# process: microservice_standard_test -# config: -# apply_manifests: -# env: *apply_env -# end2end_tests: -# metadata: -# name: e2e -# labels: -# application: kubernetes -# component: e2e -# annotations: -# zalando.org/runtime-policy: require-on-demand -# spec: -# serviceAccountName: kubernetes-e2e -# restartPolicy: Never -# containers: -# - name: e2e -# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" -# args: -# - e2e -# env: *e2e_env -# resources: -# limits: -# cpu: 2 -# memory: 8Gi -# requests: -# cpu: 2 -# memory: 8Gi -# -#- id: e2e-load-test-result -# when: -# event: pull_request -# depends_on: -# - create-cluster -# type: process -# desc: "Kubernetes e2e load tests" -# target: stups-test -# process: microservice_standard_test -# config: -# apply_manifests: -# env: *apply_env -# end2end_tests: -# metadata: -# name: e2e -# labels: -# application: kubernetes -# component: e2e -# annotations: -# zalando.org/runtime-policy: require-on-demand -# spec: -# serviceAccountName: kubernetes-e2e -# restartPolicy: Never -# containers: -# - name: e2e -# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" -# args: -# - loadtest-e2e -# env: *e2e_env -# resources: -# limits: -# cpu: 1000m -# memory: 1000Mi -# requests: -# cpu: 1000m -# memory: 1000Mi +- id: create-cluster + when: + event: pull_request + depends_on: + - build + type: process + desc: "Create cluster for Kubernetes e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: &apply_env + - name: DEPLOYMENT_PATH + value: test/e2e + - name: IAM_ROLE_ARN + value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" + - name: APPLICATION + value: kubernetes + - name: COMPONENT + value: e2e + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - create-cluster + env: &e2e_env + - name: CLUSTER_DOMAIN + value: teapot-e2e.zalan.do + - name: CLUSTER_DOMAIN_INTERNAL + value: ingress.cluster.local + - name: RESULT_BUCKET + value: "teapot-kubernetes-e2e-results" + - name: ETCD_ENDPOINTS + value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: SKIPPER_OPA_ENABLED + value: "false" + - name: HOSTED_ZONE + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "HOSTED_ZONE" + - name: REGION + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "REGION" + - name: AWS_ACCOUNT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AWS_ACCOUNT" + - name: ZMON_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ZMON_ROOT_ACCOUNT_ROLE" + - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" + - name: SESSION_MANAGER_DESTINATION_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SESSION_MANAGER_DESTINATION_ARN" + - name: APISERVER_BUSINESS_PARTNER_IDS + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "APISERVER_BUSINESS_PARTNER_IDS" + - name: LIGHTSTEP_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "LIGHTSTEP_TOKEN" + - name: OWNER + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OWNER" + - name: VPC_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "VPC_ID" + - name: EFS_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "EFS_ID" + - name: ETCD_CLIENT_CA_CERT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_CERT" + - name: ETCD_CLIENT_CA_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_KEY" + - name: ETCD_SCALYR_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_SCALYR_KEY" + - name: OKTA_AUTH_ISSUER_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" + - name: CLUSTER_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-credentials + key: "cluster-token-secret" + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + +- id: e2e-tests + when: + event: pull_request + depends_on: + - create-cluster + type: process + desc: "Kubernetes e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - e2e + env: *e2e_env + resources: + limits: + cpu: 2 + memory: 8Gi + requests: + cpu: 2 + memory: 8Gi + +- id: e2e-load-test-result + when: + event: pull_request + depends_on: + - create-cluster + type: process + desc: "Kubernetes e2e load tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - loadtest-e2e + env: *e2e_env + resources: + limits: + cpu: 1000m + memory: 1000Mi + requests: + cpu: 1000m + memory: 1000Mi + +- id: stackset-e2e-tests + when: + event: pull_request + depends_on: + - create-cluster + type: process + desc: "StackSet e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - stackset-e2e + env: *e2e_env + resources: + limits: + cpu: 2 + memory: 1Gi + requests: + cpu: 2 + memory: 1Gi + +- id: decommission-cluster + when: + event: pull_request + depends_on: + - e2e-tests + - stackset-e2e-tests + - e2e-load-test-result + type: process + desc: "Decommission cluster used for Kubernetes e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - decommission-cluster + env: *e2e_env + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi - id: create-opa-enabled-cluster when: @@ -275,15 +351,7 @@ pipeline: process: microservice_standard_test config: apply_manifests: - env: &apply_opa_enabled_env - - name: DEPLOYMENT_PATH - value: test/e2e - - name: IAM_ROLE_ARN - value: "arn:aws:iam::925511348110:role/cluster-lifecycle-manager-entrypoint" - - name: APPLICATION - value: kubernetes - - name: COMPONENT - value: e2e + env: *apply_env end2end_tests: metadata: name: e2e @@ -419,6 +487,7 @@ pipeline: requests: cpu: 500m memory: 2Gi + - id: opa-e2e-tests when: event: pull_request @@ -430,7 +499,7 @@ pipeline: process: microservice_standard_test config: apply_manifests: - env: *apply_opa_enabled_env + env: *apply_env end2end_tests: metadata: name: e2e @@ -456,55 +525,19 @@ pipeline: requests: cpu: 2 memory: 8Gi -#- id: stackset-e2e-tests -# when: -# event: pull_request -# depends_on: -# - create-cluster -# type: process -# desc: "StackSet e2e tests" -# target: stups-test -# process: microservice_standard_test -# config: -# apply_manifests: -# env: *apply_env -# end2end_tests: -# metadata: -# name: e2e -# labels: -# application: kubernetes -# component: e2e -# annotations: -# zalando.org/runtime-policy: require-on-demand -# spec: -# serviceAccountName: kubernetes-e2e -# restartPolicy: Never -# containers: -# - name: e2e -# image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" -# args: -# - stackset-e2e -# env: *e2e_env -# resources: -# limits: -# cpu: 2 -# memory: 1Gi -# requests: -# cpu: 2 -# memory: 1Gi - -- id: decommission-cluster + +- id: decommission-opa-enabled-cluster when: event: pull_request depends_on: - - opa-e2e-tests + - opa-e2e-tests type: process desc: "Decommission cluster used for Kubernetes e2e tests" target: stups-test process: microservice_standard_test config: apply_manifests: - env: *apply_opa_enabled_env + env: *apply_env end2end_tests: metadata: name: e2e @@ -517,18 +550,19 @@ pipeline: serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - decommission-cluster - env: *opa_e2e_env - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - decommission-cluster + - "true" + env: *opa_e2e_env + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi - id: deploy-event type: deploy From 0ee2d48bff3136134347937bd829023f4d67b797 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Fri, 20 Dec 2024 14:42:36 +0100 Subject: [PATCH 42/44] Refactor bash script --- test/e2e/run_e2e.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index df5892c4eb..d1f82acb06 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -102,12 +102,6 @@ if [ "$create_cluster" = true ]; then git clone "https://$CDP_TARGET_REPOSITORY" "$BASE_CFG_PATH" git -C "$BASE_CFG_PATH" reset --hard "${CDP_TARGET_COMMIT_ID}" -# if [ "$OPA_ENABLED" == "true" ]; then -# echo "Removing some cluster features to speed up the creation" -# rm -rf "$BASE_CFG_PATH/cluster/manifests/skipper-canary-controller" -# rm -rf "$BASE_CFG_PATH/cluster/manifests/prometheus" -# fi - # generate cluster.yaml # call the cluster_config.sh from base git checkout if possible if [ -f "$BASE_CFG_PATH/test/e2e/cluster_config.sh" ]; then From 20703f2656089c997b85ef700be0ef027971c0e5 Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Mon, 23 Dec 2024 09:23:38 +0100 Subject: [PATCH 43/44] Enable opa with cluster update --- delivery.yaml | 61 +++++++++++---------------------------------- test/e2e/run_e2e.sh | 19 +++----------- 2 files changed, 17 insertions(+), 63 deletions(-) diff --git a/delivery.yaml b/delivery.yaml index 77feeebeb6..7bbe20060c 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -301,50 +301,13 @@ pipeline: cpu: 2 memory: 1Gi -- id: decommission-cluster - when: - event: pull_request - depends_on: - - e2e-tests - - stackset-e2e-tests - - e2e-load-test-result - type: process - desc: "Decommission cluster used for Kubernetes e2e tests" - target: stups-test - process: microservice_standard_test - config: - apply_manifests: - env: *apply_env - end2end_tests: - metadata: - name: e2e - labels: - application: kubernetes - component: e2e - annotations: - zalando.org/runtime-policy: require-on-demand - spec: - serviceAccountName: kubernetes-e2e - restartPolicy: Never - containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - decommission-cluster - env: *e2e_env - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi - -- id: create-opa-enabled-cluster +- id: update-cluster-to-enable-opa when: event: pull_request depends_on: - - build + - e2e-tests + - e2e-load-test-result + - stackset-e2e-tests type: process desc: "Create cluster for OPA filter e2e tests" target: stups-test @@ -368,7 +331,6 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - create-cluster - - "true" env: &opa_e2e_env - name: CLUSTER_DOMAIN value: teapot-e2e.zalan.do @@ -378,8 +340,12 @@ pipeline: value: "teapot-kubernetes-e2e-results" - name: ETCD_ENDPOINTS value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: E2E_SKIP_CLUSTER_UPDATE + value: "true" - name: SKIPPER_OPA_ENABLED value: "true" + - name: FOCUS + value: "(\[Opa\])" - name: HOSTED_ZONE valueFrom: configMapKeyRef: @@ -492,7 +458,7 @@ pipeline: when: event: pull_request depends_on: - - create-opa-enabled-cluster + - update-cluster-to-enable-opa type: process desc: "OPA filter in Kubernetes e2e tests" target: stups-test @@ -516,7 +482,6 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - e2e - - "true" env: *opa_e2e_env resources: limits: @@ -526,10 +491,13 @@ pipeline: cpu: 2 memory: 8Gi -- id: decommission-opa-enabled-cluster +- id: decommission-cluster when: event: pull_request depends_on: + - e2e-tests + - stackset-e2e-tests + - e2e-load-test-result - opa-e2e-tests type: process desc: "Decommission cluster used for Kubernetes e2e tests" @@ -554,8 +522,7 @@ pipeline: image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" args: - decommission-cluster - - "true" - env: *opa_e2e_env + env: *e2e_env resources: limits: cpu: 500m diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index d1f82acb06..b7c5eaf610 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -7,7 +7,6 @@ loadtest_e2e=false stackset_e2e=false decommission_cluster=false COMMAND="${1:-"all"}" # all, create-cluster, e2e, stackset-e2e, decommission-cluster -OPA_ENABLED="${2:-false}" case "$COMMAND" in all) @@ -45,16 +44,10 @@ CDP_TARGET_REPOSITORY="${CDP_TARGET_REPOSITORY:-"github.com/zalando-incubator/ku CDP_TARGET_COMMIT_ID="${CDP_TARGET_COMMIT_ID:-"dev"}" CDP_HEAD_COMMIT_ID="${CDP_HEAD_COMMIT_ID:-"$(git describe --tags --always)"}" RESULT_BUCKET="${RESULT_BUCKET:-""}" +FOCUS="${FOCUS:-(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])}" -if [ "$OPA_ENABLED" == "true" ]; then - echo "Running with OPA specific configuration" - export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}opa"}" - export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}opa"}" -else - export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" - export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" -fi - +export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" +export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" export API_SERVER_URL="https://${LOCAL_ID}.${HOSTED_ZONE}" export INFRASTRUCTURE_ACCOUNT="aws:${AWS_ACCOUNT}" export CLUSTER_ID="${INFRASTRUCTURE_ACCOUNT}:${REGION}:${LOCAL_ID}" @@ -189,12 +182,6 @@ if [ "$e2e" = true ]; then # introduce a broken DNS record to mess with ExternalDNS # kubectl apply -f broken-dns-record.yaml - if [ "$OPA_ENABLED" == "true" ]; then - FOCUS="(\[Opa\])" - else - FOCUS="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" - fi - mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ -focus="$FOCUS" \ From 6aecc87ed82bdf51c68ba9526c56e3b6a8abe3eb Mon Sep 17 00:00:00 2001 From: nwickramasin Date: Mon, 23 Dec 2024 09:27:00 +0100 Subject: [PATCH 44/44] Fix yaml --- delivery.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/delivery.yaml b/delivery.yaml index 7bbe20060c..2f001b78a8 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -345,7 +345,7 @@ pipeline: - name: SKIPPER_OPA_ENABLED value: "true" - name: FOCUS - value: "(\[Opa\])" + value: "([Opa])" - name: HOSTED_ZONE valueFrom: configMapKeyRef: