diff --git a/cluster/manifests/skipper/rbac.yaml b/cluster/manifests/skipper/01-rbac.yaml similarity index 100% rename from cluster/manifests/skipper/rbac.yaml rename to cluster/manifests/skipper/01-rbac.yaml diff --git a/delivery.yaml b/delivery.yaml index 91b8e61804..2f001b78a8 100644 --- a/delivery.yaml +++ b/delivery.yaml @@ -80,6 +80,8 @@ pipeline: value: "teapot-kubernetes-e2e-results" - name: ETCD_ENDPOINTS value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: SKIPPER_OPA_ENABLED + value: "false" - name: HOSTED_ZONE valueFrom: configMapKeyRef: @@ -155,6 +157,26 @@ pipeline: secretKeyRef: name: kubernetes-e2e-config-secret key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" - name: CLUSTER_ADMIN_TOKEN valueFrom: secretKeyRef: @@ -279,13 +301,204 @@ pipeline: cpu: 2 memory: 1Gi +- id: update-cluster-to-enable-opa + when: + event: pull_request + depends_on: + - e2e-tests + - e2e-load-test-result + - stackset-e2e-tests + type: process + desc: "Create cluster for OPA filter e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - create-cluster + env: &opa_e2e_env + - name: CLUSTER_DOMAIN + value: teapot-e2e.zalan.do + - name: CLUSTER_DOMAIN_INTERNAL + value: ingress.cluster.local + - name: RESULT_BUCKET + value: "teapot-kubernetes-e2e-results" + - name: ETCD_ENDPOINTS + value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479" + - name: E2E_SKIP_CLUSTER_UPDATE + value: "true" + - name: SKIPPER_OPA_ENABLED + value: "true" + - name: FOCUS + value: "([Opa])" + - name: HOSTED_ZONE + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "HOSTED_ZONE" + - name: REGION + valueFrom: + configMapKeyRef: + name: kubernetes-e2e-config + key: "REGION" + - name: AWS_ACCOUNT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AWS_ACCOUNT" + - name: ZMON_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ZMON_ROOT_ACCOUNT_ROLE" + - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE" + - name: SESSION_MANAGER_DESTINATION_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SESSION_MANAGER_DESTINATION_ARN" + - name: APISERVER_BUSINESS_PARTNER_IDS + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "APISERVER_BUSINESS_PARTNER_IDS" + - name: LIGHTSTEP_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "LIGHTSTEP_TOKEN" + - name: OWNER + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OWNER" + - name: VPC_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "VPC_ID" + - name: EFS_ID + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "EFS_ID" + - name: ETCD_CLIENT_CA_CERT + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_CERT" + - name: ETCD_CLIENT_CA_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_CLIENT_CA_KEY" + - name: ETCD_SCALYR_KEY + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "ETCD_SCALYR_KEY" + - name: OKTA_AUTH_ISSUER_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "OKTA_AUTH_ISSUER_URL" + - name: STYRA_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "STYRA_TOKEN" + - name: SKIPPER_OPA_BUCKET_ARN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUCKET_ARN" + - name: SKIPPER_OPA_OBSERVABILITY_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_OBSERVABILITY_URL" + - name: SKIPPER_OPA_BUNDLES_URL + valueFrom: + secretKeyRef: + name: kubernetes-e2e-config-secret + key: "SKIPPER_OPA_BUNDLES_URL" + - name: CLUSTER_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: kubernetes-e2e-credentials + key: "cluster-token-secret" + resources: + limits: + cpu: 500m + memory: 2Gi + requests: + cpu: 500m + memory: 2Gi + +- id: opa-e2e-tests + when: + event: pull_request + depends_on: + - update-cluster-to-enable-opa + type: process + desc: "OPA filter in Kubernetes e2e tests" + target: stups-test + process: microservice_standard_test + config: + apply_manifests: + env: *apply_env + end2end_tests: + metadata: + name: e2e + labels: + application: kubernetes + component: e2e + annotations: + zalando.org/runtime-policy: require-on-demand + spec: + serviceAccountName: kubernetes-e2e + restartPolicy: Never + containers: + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - e2e + env: *opa_e2e_env + resources: + limits: + cpu: 2 + memory: 8Gi + requests: + cpu: 2 + memory: 8Gi + - id: decommission-cluster when: event: pull_request depends_on: - - e2e-tests - - stackset-e2e-tests - - e2e-load-test-result + - e2e-tests + - stackset-e2e-tests + - e2e-load-test-result + - opa-e2e-tests type: process desc: "Decommission cluster used for Kubernetes e2e tests" target: stups-test @@ -305,18 +518,18 @@ pipeline: serviceAccountName: kubernetes-e2e restartPolicy: Never containers: - - name: e2e - image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" - args: - - decommission-cluster - env: *e2e_env - resources: - limits: - cpu: 500m - memory: 1Gi - requests: - cpu: 500m - memory: 1Gi + - name: e2e + image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}" + args: + - decommission-cluster + env: *e2e_env + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 500m + memory: 1Gi - id: deploy-event type: deploy diff --git a/test/e2e/apply/secret.yaml b/test/e2e/apply/secret.yaml index aeda315268..83819035c5 100644 --- a/test/e2e/apply/secret.yaml +++ b/test/e2e/apply/secret.yaml @@ -19,3 +19,7 @@ data: ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu" OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN" SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ==" + STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A==" + SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U=" + SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA==" + SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg==" diff --git a/test/e2e/cluster_config.sh b/test/e2e/cluster_config.sh index 9bcee96c8f..82e881bd4a 100755 --- a/test/e2e/cluster_config.sh +++ b/test/e2e/cluster_config.sh @@ -47,6 +47,11 @@ clusters: karpenter_pools_enabled: "true" okta_auth_client_id: "kubernetes.cluster.teapot-e2e" teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$" + skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}" + skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}" + skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}" + skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}" + skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}" criticality_level: 1 environment: e2e id: ${CLUSTER_ID} diff --git a/test/e2e/ingress.go b/test/e2e/ingress.go index 93cab4e5c6..a80189c5c4 100644 --- a/test/e2e/ingress.go +++ b/test/e2e/ingress.go @@ -1027,3 +1027,129 @@ var _______ = describe("Ingress tests simple NLB", func() { Expect(resp.Header.Get("Request-Host")).To(Equal(hostName)) }) }) + +var ________ = describe("Ingress tests for OPA filters", func() { + f := framework.NewDefaultFramework("skipper-ingress-with-opa") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelBaseline + var ( + cs kubernetes.Interface + jig *ingress.TestJig + ) + + It("Should activate OPA filter without issue [Ingress] [Opa]", func() { + jig = ingress.NewIngressTestJig(f.ClientSet) + cs = f.ClientSet + serviceName := "skipper-ingress-test" + ns := f.Namespace.Name + hostName := fmt.Sprintf("%s-%d.%s", serviceName, time.Now().UTC().Unix(), E2EHostedZone()) + labels := map[string]string{ + "app": serviceName, + } + port := 8080 + replicas := int32(3) + targetPort := 9090 + backendContent := "mytest" + route := fmt.Sprintf(`* -> inlineContent("%s") -> `, backendContent) + waitTime := 10 * time.Minute + + // CREATE setup + // backend deployment + By("Creating a deployment with " + serviceName + " in namespace " + ns) + depl := createSkipperBackendDeployment(serviceName, ns, route, labels, int32(targetPort), replicas) + _, err := cs.AppsV1().Deployments(ns).Create(context.TODO(), depl, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + By("Creating service " + serviceName + " in namespace " + ns) + service := createServiceTypeClusterIP(serviceName, labels, port, targetPort) + _, err = cs.CoreV1().Services(ns).Create(context.TODO(), service, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ing := createIngress(serviceName, hostName, ns, "/", netv1.PathTypeImplementationSpecific, labels, nil, port) + ingressCreate, err := cs.NetworkingV1().Ingresses(ns).Create(context.TODO(), ing, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + addr, err := jig.WaitForIngressAddress(context.TODO(), cs, ns, ingressCreate.Name, waitTime) + framework.ExpectNoError(err) + + _, err = cs.NetworkingV1().Ingresses(ns).Get(context.TODO(), ing.Name, metav1.GetOptions{ResourceVersion: "0"}) + framework.ExpectNoError(err) + + // skipper http -> https redirect + By("Waiting for skipper route to default redirect from http to https, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "http", waitTime, isRedirect, true) + framework.ExpectNoError(err) + + // ALB ready + By("Waiting for ALB to create endpoint " + addr + " and skipper route, to see that our ingress-controller and skipper works") + err = waitForResponse(addr, "https", waitTime, isNotFound, true) + framework.ExpectNoError(err) + + // DNS ready + By("Waiting for DNS to see that external-dns and skipper route to service and pod works") + err = waitForResponse(hostName, "https", waitTime, isSuccess, false) + framework.ExpectNoError(err) + + // Test that we get content from the default ingress + By("By checking the content of the reply we see that the ingress stack works") + rt, quit := createHTTPRoundTripper() + defer func() { + quit <- struct{}{} + }() + url := "https://" + hostName + "/" + req, err := http.NewRequest("GET", url, nil) + framework.ExpectNoError(err) + resp, err := rt.RoundTrip(req) + framework.ExpectNoError(err) + s, err := getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + + // Start actual ingress tests + // Test ingress Filters: opaAuthorizeRequest + + /** + ## The Rule looks like below. + ## Reference https://github.bus.zalan.do/corporate-iam/styra-smoketest-policies/blob/main/bundle/policy/ingress/rules.rego + default allow := false + + allow if { + input.attributes.request.http.method == "GET" + auth_header_val := input.attributes.request.http.headers.authorization + startswith(auth_header_val, "Basic ") + token := substring(auth_header_val, count("Basic "), -1) + token == "valid_token" + } + */ + path := "/" + opaPolicyName := "styra-smoketest" + updatedIng := updateIngress(ingressCreate.ObjectMeta.Name, + ingressCreate.ObjectMeta.Namespace, + hostName, + serviceName, + path, + netv1.PathTypeImplementationSpecific, + ingressCreate.ObjectMeta.Labels, + map[string]string{ + "zalando.org/skipper-filter": fmt.Sprintf(`opaAuthorizeRequest("%s")`, opaPolicyName), + }, + port, + ) + ingressUpdate, err := cs.NetworkingV1().Ingresses(ingressCreate.ObjectMeta.Namespace).Update(context.TODO(), updatedIng, metav1.UpdateOptions{}) + framework.ExpectNoError(err) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 403 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + time.Sleep(20 * time.Second) // wait for routing change propagation + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusForbidden) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusForbidden)) + + By(fmt.Sprintf("Waiting for ingress %s/%s we wait to get a 200 with opaAuthorizeRequest %s policy", ingressUpdate.Namespace, ingressUpdate.Name, opaPolicyName)) + req.Header.Set("Authorization", "Basic valid_token") //Authorized request + resp, err = getAndWaitResponse(rt, req, 10*time.Second, http.StatusOK) + framework.ExpectNoError(err) + Expect(resp.StatusCode).To(Equal(http.StatusOK)) + s, err = getBody(resp) + framework.ExpectNoError(err) + Expect(s).To(Equal(backendContent)) + }) +}) diff --git a/test/e2e/run_e2e.sh b/test/e2e/run_e2e.sh index eb8b2f5538..b7c5eaf610 100755 --- a/test/e2e/run_e2e.sh +++ b/test/e2e/run_e2e.sh @@ -44,6 +44,7 @@ CDP_TARGET_REPOSITORY="${CDP_TARGET_REPOSITORY:-"github.com/zalando-incubator/ku CDP_TARGET_COMMIT_ID="${CDP_TARGET_COMMIT_ID:-"dev"}" CDP_HEAD_COMMIT_ID="${CDP_HEAD_COMMIT_ID:-"$(git describe --tags --always)"}" RESULT_BUCKET="${RESULT_BUCKET:-""}" +FOCUS="${FOCUS:-(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])}" export CLUSTER_ALIAS="${CLUSTER_ALIAS:-"e2e-${CDP_BUILD_VERSION}"}" export LOCAL_ID="${LOCAL_ID:-"e2e-${CDP_BUILD_VERSION}"}" @@ -183,7 +184,7 @@ if [ "$e2e" = true ]; then mkdir -p junit_reports ginkgo -procs=25 -flake-attempts=2 \ - -focus="(\[Conformance\]|\[StatefulSetBasic\]|\[Feature:StatefulSet\]\s\[Slow\].*mysql|\[Zalando\])" \ + -focus="$FOCUS" \ -skip="(\[Serial\]|validates.that.there.is.no.conflict.between.pods.with.same.hostPort.but.different.hostIP.and.protocol|Should.create.gradual.traffic.routes)" \ "e2e.test" -- \ -delete-namespace-on-failure=false \