Add ingress test for opaAuthorizeRequest filter

delivery.yaml

@@ -155,6 +155,26 @@ pipeline:
               secretKeyRef:
                 name: kubernetes-e2e-config-secret
                 key: "OKTA_AUTH_ISSUER_URL"
+          - name: STYRA_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "STYRA_TOKEN"
+          - name: SKIPPER_OPA_BUCKET_ARN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUCKET_ARN"
+          - name: SKIPPER_OPA_OBSERVABILITY_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_OBSERVABILITY_URL"
+          - name: SKIPPER_OPA_BUNDLES_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUNDLES_URL"
           - name: CLUSTER_ADMIN_TOKEN
             valueFrom:
               secretKeyRef:

test/e2e/apply/secret.yaml

@@ -19,3 +19,7 @@ data:
   ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
   OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
   SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
+  STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A=="
+  SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
+  SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
+  SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="

test/e2e/cluster_config.sh

@@ -47,6 +47,11 @@ clusters:
     karpenter_pools_enabled: "true"
     okta_auth_client_id: "kubernetes.cluster.teapot-e2e"
     teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$"
+    skipper_open_policy_agent_enabled: "true"
+    skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}"
+    skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}"
+    skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}"
+    skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}"
   criticality_level: 1
   environment: e2e
   id: ${CLUSTER_ID}

test/e2e/ingress.go

@@ -315,7 +315,135 @@ var __ = describe("Ingress tests simple", func() {
 	})
 })
 
-var ___ = describe("Ingress tests paths", func() {
+var ___ = describe("Ingress tests for OPA filters", func() {
+	f := framework.NewDefaultFramework("skipper-ingress-with-opa")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
+	var (
+		cs  kubernetes.Interface
+		jig *ingress.TestJig
+	)
+
+	time.Sleep(120 * time.Second) // see weather waiting helps with AWS_ROLE_ARN
+
+	It("Should activate OPA filter without issue [Ingress] [ZalandoIAM]", func() {
+		jig = ingress.NewIngressTestJig(f.ClientSet)
+		cs = f.ClientSet
+		serviceName := "skipper-ingress-test"
+		ns := f.Namespace.Name
+		hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone())
+		labels := map[string]string{
+			"app": serviceName,
+		}
+		port := 8080
+		replicas := int32(3)
+		targetPort := 9090
+		backendContent := "mytest"
+		route := fmt.Sprintf(`* -> inlineContent("%s") -> <shunt>`, backendContent)
+		waitTime := 10 * time.Minute
+
+		// CREATE setup
+		// backend deployment
+		By("Creating a deployment with " + serviceName + " in namespace " + ns)
+		depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas)
+		_, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		By("Creating service " + serviceName + " in namespace " + ns)
+		service := createServiceTypeClusterIP(serviceName, labels, port, targetPort)
+		_, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port)
+		ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{})
+		framework.ExpectNoError(err)
+
+		addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime)
+		framework.ExpectNoError(err)
+
+		_, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"})
+		framework.ExpectNoError(err)
+
+		// skipper http -> https redirect
+		By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works")
+		err = waitForResponse(addr, "http", waitTime, isRedirect, true)
+		framework.ExpectNoError(err)
+
+		// ALB ready
+		By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works")
+		err = waitForResponse(addr, "https", waitTime, isNotFound, true)
+		framework.ExpectNoError(err)
+
+		// DNS ready
+		By("Waiting for DNS to see that external-dns and skipper route to service and pod works")
+		err = waitForResponse(hostName, "https", waitTime, isSuccess, false)
+		framework.ExpectNoError(err)
+
+		// Test that we get content from the default ingress
+		By("By checking the content of the reply we see that the ingress stack works")
+		rt, quit := createHTTPRoundTripper()
+		defer func() {
+			quit <- struct{}{}
+		}()
+		url := "https://" + hostName + "/"
+		req, err := http.NewRequest("GET", url, nil)
+		framework.ExpectNoError(err)
+		resp, err := rt.RoundTrip(req)
+		framework.ExpectNoError(err)
+		s, err := getBody(resp)
+		framework.ExpectNoError(err)
+		Expect(s).To(Equal(backendContent))
+
+		// Start actual ingress tests
+		// Test ingress Filters: opaAuthorizeRequest
+
+		/**
+		## The Rule looks like below.
+		## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego
+		default allow := false
+
+		allow if {
+		  input.attributes.request.http.method == "GET"
+		  auth_header_val := input.attributes.request.http.headers.authorization
+		  startswith(auth_header_val, "Basic ")
+		  token := substring(auth_header_val, count("Basic "), -1)
+		  token == "valid_token"
+		}
+		*/
+		path := "/"
+		opaPolicyName := "styra-smoketest"
+		updatedIng := updateIngress(ingressCreate.ObjectMeta.Name,
+			ingressCreate.ObjectMeta.Namespace,
+			hostName,
+			serviceName,
+			path,
+			netv1.PathTypeImplementationSpecific,
+			ingressCreate.ObjectMeta.Labels,
+			map[string]string{
+				"zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName),
+			},
+			port,
+		)
+		ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{})
+		framework.ExpectNoError(err)
+
+		By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		time.Sleep(20 * time.Second) // wait for routing change propagation
+		resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusForbidden))
+
+		By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName))
+		req.Header.Set("Authorization", "Basic valid_token") //Authorized request
+		resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK)
+		framework.ExpectNoError(err)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		s, err = getBody(resp)
+		framework.ExpectNoError(err)
+		Expect(s).To(Equal(backendContent))
+	})
+})
+
+var ____ = describe("Ingress tests paths", func() {
 	f := framework.NewDefaultFramework("skipper-ingress-paths")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var (
@@ -488,7 +616,7 @@ var ___ = describe("Ingress tests paths", func() {
 	})
 })
 
-var ____ = describe("Ingress tests custom routes", func() {
+var _____ = describe("Ingress tests custom routes", func() {
 	f := framework.NewDefaultFramework("skipper-ingress-custom")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var (
@@ -611,7 +739,7 @@ var ____ = describe("Ingress tests custom routes", func() {
 	})
 })
 
-var _____ = describe("Ingress tests paths", func() {
+var ______ = describe("Ingress tests paths", func() {
 	f := framework.NewDefaultFramework("skipper-ingress-paths")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var (
@@ -800,7 +928,7 @@ var _____ = describe("Ingress tests paths", func() {
 	})
 })
 
-var ______ = describe("Ingress tests custom routes", func() {
+var _______ = describe("Ingress tests custom routes", func() {
 	f := framework.NewDefaultFramework("skipper-ingress-custom")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var (
@@ -923,7 +1051,7 @@ var ______ = describe("Ingress tests custom routes", func() {
 	})
 })
 
-var _______ = describe("Ingress tests simple NLB", func() {
+var ________ = describe("Ingress tests simple NLB", func() {
 	f := framework.NewDefaultFramework("skipper-ingress-simple-nlb")
 	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline
 	var (

test/e2e/run_e2e.sh

@@ -183,7 +183,7 @@ if [ "$e2e" = true ]; then
 
     mkdir -p junit_reports
     ginkgo -procs=25 -flake-attempts=2 \
-        -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \
+        -focus="(\[ZalandoIAM\])" \
         -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \
         "e2e.test" -- \
         -delete-namespace-on-failure=false \