From 4bcd20888d0d1396b9f41fae54ed7a39e1c311d4 Mon Sep 17 00:00:00 2001 From: red-hat-konflux Date: Tue, 5 Nov 2024 01:53:49 +0000 Subject: [PATCH 01/10] Red Hat Konflux update rapidast Signed-off-by: red-hat-konflux --- .tekton/rapidast-pull-request.yaml | 474 +++++++++++++++++++++++++++++ .tekton/rapidast-push.yaml | 471 ++++++++++++++++++++++++++++ 2 files changed, 945 insertions(+) create mode 100644 .tekton/rapidast-pull-request.yaml create mode 100644 .tekton/rapidast-push.yaml diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml new file mode 100644 index 00000000..9d774d10 --- /dev/null +++ b/.tekton/rapidast-pull-request.yaml @@ -0,0 +1,474 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RedHatProductSecurity/rapidast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch + == "development" + creationTimestamp: null + labels: + appstudio.openshift.io/application: rapidast + appstudio.openshift.io/component: rapidast + pipelines.appstudio.openshift.io/type: build + name: rapidast-on-pull-request + namespace: secaut-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/secaut-tenant/rapidast:on-pr-{{revision}} + - name: image-expires-after + value: 5d + - name: dockerfile + value: containerize/Containerfile + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:52f8b96b96ce4203d4b74d850a85f963125bf8eef0683ea5acdd80818d335a28 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:2a0c67ea7d5d82b4ec47930c12397f94b3af0b3855d8e5ad9f6e088c93e42bf0 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:327d745a58c1589b0ff196ed526d12a8a0a20ae22fd1c9dd1577b850a977dc3b + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:28fee4bf5da87f2388c973d9336086749cad8436003f9a514e22ac99735e056b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:7e99a122bc9e84fd9fb29062e825d3345177337d2448dcb50324f86ec5560c7a + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:a94b6523ba0b691dc276e37594321c2eff3594d2753014e5c920803b47627df1 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:e32feb2c815116730917fe5665d9f003e53f2e1718f60bcbabf0ab3abad5d7d4 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} diff --git a/.tekton/rapidast-push.yaml b/.tekton/rapidast-push.yaml new file mode 100644 index 00000000..22f9131b --- /dev/null +++ b/.tekton/rapidast-push.yaml @@ -0,0 +1,471 @@ +apiVersion: tekton.dev/v1 +kind: PipelineRun +metadata: + annotations: + build.appstudio.openshift.io/repo: https://github.com/RedHatProductSecurity/rapidast?rev={{revision}} + build.appstudio.redhat.com/commit_sha: '{{revision}}' + build.appstudio.redhat.com/target_branch: '{{target_branch}}' + pipelinesascode.tekton.dev/max-keep-runs: "3" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch + == "development" + creationTimestamp: null + labels: + appstudio.openshift.io/application: rapidast + appstudio.openshift.io/component: rapidast + pipelines.appstudio.openshift.io/type: build + name: rapidast-on-push + namespace: secaut-tenant +spec: + params: + - name: git-url + value: '{{source_url}}' + - name: revision + value: '{{revision}}' + - name: output-image + value: quay.io/redhat-user-workloads/secaut-tenant/rapidast:{{revision}} + - name: dockerfile + value: containerize/Containerfile + pipelineSpec: + description: | + This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. + + _Uses `buildah` to create a container image leveraging [trusted artifacts](https://konflux-ci.dev/architecture/ADR/0036-trusted-artifacts.html). It also optionally creates a source image and runs some build-time tests. Information is shared between tasks using OCI artifacts instead of PVCs. EC will pass the [`trusted_task.trusted`](https://enterprisecontract.dev/docs/ec-policies/release_policy.html#trusted_task__trusted) policy as long as all data used to build the artifact is generated from trusted tasks. + This pipeline is pushed as a Tekton bundle to [quay.io](https://quay.io/repository/konflux-ci/tekton-catalog/pipeline-docker-build-oci-ta?tab=tags)_ + finally: + - name: show-sbom + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + taskRef: + params: + - name: name + value: show-sbom + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-show-sbom:0.1@sha256:52f8b96b96ce4203d4b74d850a85f963125bf8eef0683ea5acdd80818d335a28 + - name: kind + value: task + resolver: bundles + params: + - description: Source Repository URL + name: git-url + type: string + - default: "" + description: Revision of the Source Repository + name: revision + type: string + - description: Fully Qualified Output Image + name: output-image + type: string + - default: . + description: Path to the source code of an application's component from where + to build image. + name: path-context + type: string + - default: Dockerfile + description: Path to the Dockerfile inside the context specified by parameter + path-context + name: dockerfile + type: string + - default: "false" + description: Force rebuild image + name: rebuild + type: string + - default: "false" + description: Skip checks against built image + name: skip-checks + type: string + - default: "false" + description: Execute the build with network isolation + name: hermetic + type: string + - default: "" + description: Build dependencies to be prefetched by Cachi2 + name: prefetch-input + type: string + - default: "" + description: Image tag expiration time, time values could be something like + 1h, 2d, 3w for hours, days, and weeks, respectively. + name: image-expires-after + - default: "false" + description: Build a source image. + name: build-source-image + type: string + - default: "false" + description: Add built image into an OCI image index + name: build-image-index + type: string + - default: [] + description: Array of --build-arg values ("arg=value" strings) for buildah + name: build-args + type: array + - default: "" + description: Path to a file with build arguments for buildah, see https://www.mankier.com/1/buildah-build#--build-arg-file + name: build-args-file + type: string + results: + - description: "" + name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - description: "" + name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - description: "" + name: CHAINS-GIT_URL + value: $(tasks.clone-repository.results.url) + - description: "" + name: CHAINS-GIT_COMMIT + value: $(tasks.clone-repository.results.commit) + tasks: + - name: init + params: + - name: image-url + value: $(params.output-image) + - name: rebuild + value: $(params.rebuild) + - name: skip-checks + value: $(params.skip-checks) + taskRef: + params: + - name: name + value: init + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-init:0.2@sha256:092c113b614f6551113f17605ae9cb7e822aa704d07f0e37ed209da23ce392cc + - name: kind + value: task + resolver: bundles + - name: clone-repository + params: + - name: url + value: $(params.git-url) + - name: revision + value: $(params.revision) + - name: ociStorage + value: $(params.output-image).git + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - init + taskRef: + params: + - name: name + value: git-clone-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-git-clone-oci-ta:0.1@sha256:d1e63ec00bed1c9f0f571fa76b4da570be49a7c255c610544a461495230ba1b1 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + workspaces: + - name: basic-auth + workspace: git-auth + - name: prefetch-dependencies + params: + - name: input + value: $(params.prefetch-input) + - name: SOURCE_ARTIFACT + value: $(tasks.clone-repository.results.SOURCE_ARTIFACT) + - name: ociStorage + value: $(params.output-image).prefetch + - name: ociArtifactExpiresAfter + value: $(params.image-expires-after) + runAfter: + - clone-repository + taskRef: + params: + - name: name + value: prefetch-dependencies-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-prefetch-dependencies-oci-ta:0.1@sha256:621b13ab4a01a366a2b1d8403cf06b2b7418afd926d13678c4432858514407d3 + - name: kind + value: task + resolver: bundles + workspaces: + - name: git-basic-auth + workspace: git-auth + - name: netrc + workspace: netrc + - name: build-container + params: + - name: IMAGE + value: $(params.output-image) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: HERMETIC + value: $(params.hermetic) + - name: PREFETCH_INPUT + value: $(params.prefetch-input) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: BUILD_ARGS + value: + - $(params.build-args[*]) + - name: BUILD_ARGS_FILE + value: $(params.build-args-file) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - prefetch-dependencies + taskRef: + params: + - name: name + value: buildah-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-buildah-oci-ta:0.2@sha256:2a0c67ea7d5d82b4ec47930c12397f94b3af0b3855d8e5ad9f6e088c93e42bf0 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-image-index + params: + - name: IMAGE + value: $(params.output-image) + - name: COMMIT_SHA + value: $(tasks.clone-repository.results.commit) + - name: IMAGE_EXPIRES_AFTER + value: $(params.image-expires-after) + - name: ALWAYS_BUILD_INDEX + value: $(params.build-image-index) + - name: IMAGES + value: + - $(tasks.build-container.results.IMAGE_URL)@$(tasks.build-container.results.IMAGE_DIGEST) + runAfter: + - build-container + taskRef: + params: + - name: name + value: build-image-index + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-build-image-index:0.1@sha256:327d745a58c1589b0ff196ed526d12a8a0a20ae22fd1c9dd1577b850a977dc3b + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - name: build-source-image + params: + - name: BINARY_IMAGE + value: $(params.output-image) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: source-build-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-source-build-oci-ta:0.1@sha256:261f075fd5a096f7b28a999b505136b2a3a5aef390087148b3131fd3ec295db3 + - name: kind + value: task + resolver: bundles + when: + - input: $(tasks.init.results.build) + operator: in + values: + - "true" + - input: $(params.build-source-image) + operator: in + values: + - "true" + - name: deprecated-base-image-check + params: + - name: IMAGE_URL + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: deprecated-image-check + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-deprecated-image-check:0.4@sha256:b4f9599f5770ea2e6e4d031224ccc932164c1ecde7f85f68e16e99c98d754003 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clair-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clair-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clair-scan:0.2@sha256:28fee4bf5da87f2388c973d9336086749cad8436003f9a514e22ac99735e056b + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: ecosystem-cert-preflight-checks + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: ecosystem-cert-preflight-checks + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-ecosystem-cert-preflight-checks:0.1@sha256:5131cce0f93d0b728c7bcc0d6cee4c61d4c9f67c6d619c627e41e3c9775b497d + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: sast-snyk-check + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + - name: CACHI2_ARTIFACT + value: $(tasks.prefetch-dependencies.results.CACHI2_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: sast-snyk-check-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-sast-snyk-check-oci-ta:0.2@sha256:7e99a122bc9e84fd9fb29062e825d3345177337d2448dcb50324f86ec5560c7a + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: clamav-scan + params: + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: clamav-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-clamav-scan:0.1@sha256:a94b6523ba0b691dc276e37594321c2eff3594d2753014e5c920803b47627df1 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + - name: apply-tags + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: apply-tags + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-apply-tags:0.1@sha256:f485e250fb060060892b633c495a3d7e38de1ec105ae1be48608b0401530ab2c + - name: kind + value: task + resolver: bundles + - name: push-dockerfile + params: + - name: IMAGE + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: IMAGE_DIGEST + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + - name: DOCKERFILE + value: $(params.dockerfile) + - name: CONTEXT + value: $(params.path-context) + - name: SOURCE_ARTIFACT + value: $(tasks.prefetch-dependencies.results.SOURCE_ARTIFACT) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: push-dockerfile-oci-ta + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-push-dockerfile-oci-ta:0.1@sha256:e32feb2c815116730917fe5665d9f003e53f2e1718f60bcbabf0ab3abad5d7d4 + - name: kind + value: task + resolver: bundles + - name: rpms-signature-scan + params: + - name: image-url + value: $(tasks.build-image-index.results.IMAGE_URL) + - name: image-digest + value: $(tasks.build-image-index.results.IMAGE_DIGEST) + runAfter: + - build-image-index + taskRef: + params: + - name: name + value: rpms-signature-scan + - name: bundle + value: quay.io/konflux-ci/tekton-catalog/task-rpms-signature-scan:0.2@sha256:7aa4d3c95e2b963e82fdda392f7cb3d61e3dab035416cf4a3a34e43cf3c9c9b8 + - name: kind + value: task + resolver: bundles + when: + - input: $(params.skip-checks) + operator: in + values: + - "false" + workspaces: + - name: git-auth + optional: true + - name: netrc + optional: true + taskRunTemplate: {} + workspaces: + - name: git-auth + secret: + secretName: '{{ git_auth_secret }}' +status: {} From ebf153e21f9e604cba0dc39ff1654f9cee64ccd0 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 29 Oct 2024 14:47:03 +1000 Subject: [PATCH 02/10] add e2e tests --- .github/workflows/run-tests.yml | 2 +- .pre-commit-config.yaml | 5 +- .tekton/integration-test.yaml | 23 ++ .../manifests/integration-test-pipeline.yaml | 211 ++++++++++++++++++ .../rapidast-oobtkube-configmap.yaml | 46 ++++ .../manifests/rapidast-oobtkube-pod.yaml | 37 +++ .../manifests/rapidast-oobtkube-service.yaml | 19 ++ .../manifests/rapidast-trivy-configmap.yaml | 40 ++++ e2e-tests/manifests/rapidast-trivy-pod.yaml | 38 ++++ .../manifests/rapidast-vapi-configmap.yaml | 41 ++++ e2e-tests/manifests/rapidast-vapi-pod.yaml | 52 +++++ .../manifests/task-controller-deployment.yaml | 34 +++ e2e-tests/manifests/vapi-deployment.yaml | 29 +++ e2e-tests/manifests/vapi-service.yaml | 19 ++ e2e-tests/test_integration.py | 200 +++++++++++++++++ requirements-dev.txt | 2 + 16 files changed, 795 insertions(+), 3 deletions(-) create mode 100644 .tekton/integration-test.yaml create mode 100644 e2e-tests/manifests/integration-test-pipeline.yaml create mode 100644 e2e-tests/manifests/rapidast-oobtkube-configmap.yaml create mode 100644 e2e-tests/manifests/rapidast-oobtkube-pod.yaml create mode 100644 e2e-tests/manifests/rapidast-oobtkube-service.yaml create mode 100644 e2e-tests/manifests/rapidast-trivy-configmap.yaml create mode 100644 e2e-tests/manifests/rapidast-trivy-pod.yaml create mode 100644 e2e-tests/manifests/rapidast-vapi-configmap.yaml create mode 100644 e2e-tests/manifests/rapidast-vapi-pod.yaml create mode 100644 e2e-tests/manifests/task-controller-deployment.yaml create mode 100644 e2e-tests/manifests/vapi-deployment.yaml create mode 100644 e2e-tests/manifests/vapi-service.yaml create mode 100644 e2e-tests/test_integration.py diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 96005c4c..b43b9b32 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -26,7 +26,7 @@ jobs: pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt - name: Test with pytest run: | - pytest + pytest tests - name: Lint with pre-commit hook run: | pre-commit run --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 88488353..13608cee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,6 +37,7 @@ repos: # W0603 - Using the global statement # C0114,C0115,C0116 - docstring checks. Disabled because of pydocstyle checks # W0107 - unnecessary pass + # W0511: fixme # W0702: No exception type(s) specified (bare-except) # R0801: Similar lines in 2 files. Disabled because it flags any file even those which are unrelated # R1705: Unnecessary "elif" after "return", remove the leading "el" from "elif" (no-else-return) @@ -51,6 +52,6 @@ repos: args: - --max-line-length=120 - --min-public-methods=0 - - --good-names=q,f,fp,i,e - - --disable=E0401,W1201,W1203,C0114,C0115,C0116,C0411,W0107,W0702,R0801,R1705,R1710 + - --good-names=w,q,f,fp,i,e + - --disable=E0401,W1201,W1203,C0114,C0115,C0116,C0411,W0107,W0511,W0702,R0801,R1705,R1710 language_version: python3 diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml new file mode 100644 index 00000000..9af4aab1 --- /dev/null +++ b/.tekton/integration-test.yaml @@ -0,0 +1,23 @@ +apiVersion: appstudio.redhat.com/v1beta2 +kind: IntegrationTestScenario +metadata: + name: experiment-test + annotations: + pipelinesascode.tekton.dev/on-event: "[pull_request, push]" + pipelinesascode.tekton.dev/on-target-branch: "[main, development]" + labels: + test.appstudio.openshift.io/optional: "true" +spec: + application: rapidast + contexts: + - description: Application testing + name: application + resolverRef: + params: + - name: url + value: https://github.com/sfowl/rapidast + - name: revision + value: "{{ revision }}" + - name: pathInRepo + value: e2e-tests/manifests/integration-test-pipeline.yaml + resolver: git diff --git a/e2e-tests/manifests/integration-test-pipeline.yaml b/e2e-tests/manifests/integration-test-pipeline.yaml new file mode 100644 index 00000000..121be3fe --- /dev/null +++ b/e2e-tests/manifests/integration-test-pipeline.yaml @@ -0,0 +1,211 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: rapidast-e2e +spec: + params: + - name: repo_url + default: github.com/sfowl/rapidast + - name: revision + default: "development" + - name: image + default: quay.io/redhatproductsecurity/rapidast:latest + - description: 'Snapshot of the application' + name: SNAPSHOT + default: |- + '{ + "components": [ + { + "name":"rapidast", + "containerImage": "quay.io/redhatproductsecurity/rapidast:latest", + "source":{ + "git":{ + "url":"git@github.com:sfowl/rapidast.git", + "revision":"development", + } + } + } + ] + }' + type: string + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: provision-eaas-space + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/provision-env-with-ephemeral-namespace/0.1/provision-env-with-ephemeral-namespace.yaml + params: + - name: KONFLUXNAMESPACE + value: $(context.pipelineRun.namespace) + - name: PIPELINERUN_NAME + value: $(context.pipelineRun.name) + - name: PIPELINERUN_UID + value: $(context.pipelineRun.uid) + + # - name: provision-cluster + # runAfter: + # - provision-eaas-space + # taskSpec: + # results: + # - name: clusterName + # value: "$(steps.create-cluster.results.clusterName)" + # volumes: + # - name: credentials + # emptyDir: {} + # steps: + # - name: get-supported-versions + # ref: + # resolver: git + # params: + # - name: url + # value: https://github.com/konflux-ci/build-definitions.git + # - name: revision + # value: main + # - name: pathInRepo + # value: stepactions/eaas-get-supported-ephemeral-cluster-versions/0.1/eaas-get-supported-ephemeral-cluster-versions.yaml + # params: + # - name: eaasSpaceSecretRef + # value: $(tasks.provision-eaas-space.results.secretRef) + # - name: pick-version + # ref: + # resolver: git + # params: + # - name: url + # value: https://github.com/konflux-ci/build-definitions.git + # - name: revision + # value: main + # - name: pathInRepo + # value: stepactions/eaas-get-latest-openshift-version-by-prefix/0.1/eaas-get-latest-openshift-version-by-prefix.yaml + # params: + # - name: prefix + # value: "$(steps.get-supported-versions.results.versions[0])." + # - name: create-cluster + # ref: + # resolver: git + # params: + # - name: url + # value: https://github.com/konflux-ci/build-definitions.git + # - name: revision + # value: main + # - name: pathInRepo + # value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml + # params: + # - name: eaasSpaceSecretRef + # value: $(tasks.provision-eaas-space.results.secretRef) + # - name: version + # value: "$(steps.pick-version.results.version)" + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + # + # - name: clone-repository + # runAfter: + # - parse-metadata + # params: + # - name: url + # value: "$(tasks.parse-metadata.results.source-git-url)" + # - name: revision + # value: "$(tasks.parse-metadata.results.source-git-revision)" + # taskRef: + # name: git-clone + # workspaces: + # - name: output + # workspace: source + + - name: run-e2e-tests + runAfter: + - provision-eaas-space + taskSpec: + volumes: + - name: credentials + emptyDir: {} + results: + - name: TEST_RESULTS + description: e2e test results + steps: + # - name: get-kubeconfig + # ref: + # resolver: git + # params: + # - name: url + # value: https://github.com/konflux-ci/build-definitions.git + # - name: revision + # value: main + # - name: pathInRepo + # value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml + # params: + # - name: eaasSpaceSecretRef + # value: $(tasks.provision-eaas-space.results.secretRef) + # - name: clusterName + # value: "$(tasks.provision-cluster.results.clusterName)" + # - name: credentials + # value: credentials + + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + - name: clone-repository + image: quay.io/konflux-ci/git-clone:latest + script: | + git config --global --add safe.directory /workspace + git clone "$(tasks.parse-metadata.results.source-git-url)" /workspace + pushd /workspace + # git checkout "$(tasks.parse-metadata.results.source-git-revision)" + # XXX Temporary hardcoded repo + commit ! + git checkout test-e2e-test + + - name: test + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: /tmp/kubeconfig + # value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" + - name: KUBECONFIG_VALUE + valueFrom: + secretKeyRef: + name: $(tasks.provision-eaas-space.results.secretRef) + key: kubeconfig + - name: RAPIDAST_CLEANUP + value: "false" # namespace will be cleanedup automatically + - name: RAPIDAST_IMAGE + value: $(tasks.parse-metadata.results.component-container-image) + - name: RAPIDAST_SERVICEACCOUNT + value: namespace-manager # created by provision-env-with-ephemeral-namespace + workingDir: /workspace + volumeMounts: + - name: credentials + mountPath: /credentials + script: | + #!/bin/bash -ex + + echo "$KUBECONFIG_VALUE" > "$KUBECONFIG" + oc whoami + + yum install -y python3.12 + python3.12 -m ensurepip + pip3 install -r requirements.txt -r requirements-dev.txt + pytest -s e2e-tests --json-report --json-report-summary --json-report-file $(results.TEST_RESULTS.path) + cat $(results.TEST_RESULTS.path) diff --git a/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml b/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml new file mode 100644 index 00000000..0648aecf --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-configmap.yaml @@ -0,0 +1,46 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "oobttest" + + general: + container: + # currently supported: `podman` and `none` + type: "none" + + scanners: + generic_1: + # results: + # An absolute path to file or directory where results are stored on the host. + # if it is "*stdout" or unspecified, the command's standard output will be selected + results: "/tmp/oobtkube.sarif.json" # if None or "*stdout", the command's standard output is selected + # toolDir: scanners/generic/tools + inline: "python3 oobtkube.py --log-level debug -d 60 -p 6000 -i rapidast-oobtkube -f /opt/rapidast/config/cr_example.yaml | tee /tmp/oobtkube.sarif.json" + # XXX using tekton Task because it has: + # - a resource type that has a .spec field (required by oobtkube script) + # - fields in .spec that are arbitrary string (.spec.description) + # this could be replaced later with something more common like a ConfigMap, + # once oobtkube can test non .spec values + cr_example.yaml: |+ + apiVersion: tekton.dev/v1 + kind: Task + metadata: + name: vulnerable + spec: + description: foobar + params: + - name: foo + type: string + steps: + - image: foo + name: foo + script: foo + +kind: ConfigMap +metadata: + name: rapidast-oobtkube diff --git a/e2e-tests/manifests/rapidast-oobtkube-pod.yaml b/e2e-tests/manifests/rapidast-oobtkube-pod.yaml new file mode 100644 index 00000000..bd344915 --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-pod.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-oobtkube + labels: + app: rapidast-oobtkube +spec: + containers: + - command: + - bash + - -c + - ./rapidast.py + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-oobtkube + resources: + limits: + cpu: 0.5 + memory: 1Gi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + serviceAccountName: ${SERVICEACCOUNT} # oobtkube needs perms to query API server + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-oobtkube + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never diff --git a/e2e-tests/manifests/rapidast-oobtkube-service.yaml b/e2e-tests/manifests/rapidast-oobtkube-service.yaml new file mode 100644 index 00000000..5cf4ccda --- /dev/null +++ b/e2e-tests/manifests/rapidast-oobtkube-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: rapidast-oobtkube + name: rapidast-oobtkube +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 6000 + protocol: TCP + targetPort: 6000 + selector: + app: rapidast-oobtkube + sessionAffinity: None + type: ClusterIP diff --git a/e2e-tests/manifests/rapidast-trivy-configmap.yaml b/e2e-tests/manifests/rapidast-trivy-configmap.yaml new file mode 100644 index 00000000..f0fcdca3 --- /dev/null +++ b/e2e-tests/manifests/rapidast-trivy-configmap.yaml @@ -0,0 +1,40 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + # WARNING: `configVersion` indicates the schema version of the config file. + # This value tells RapiDAST what schema should be used to read this configuration. + # Therefore you should only change it if you update the configuration to a newer schema + # It is intended to keep backward compatibility (newer RapiDAST running an older config) + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "my-cluster" + + # `scanners' is a section that configures scanning options + scanners: + generic_trivy: + # results: + # An absolute path to file or directory where results are stored on the host. + # if it is "*stdout" or unspecified, the command's standard output will be selected + # When container.type is 'podman', this needs to be used along with the container.volumes configuration below + # If the result needs to be sent to DefectDojo, this must be a SARIF format file + #results: "/path/to/results" + + # Example: scan a k8s cluster for misconfiguration issue + # - See https://aquasecurity.github.io/trivy/v0.49/docs/target/kubernetes/ for more information on 'trivy k8s' scan + # - scanners/generic/tools/convert_trivy_k8s_to_sarif.py converts the Trivy json result to the SARIF format + # 'inline' is used when container.type is not 'podman' + # 'toolDir' specifies the default directory where inline scripts are located + #toolDir: scanners/generic/tools + inline: "trivy k8s -n $(cat /run/secrets/kubernetes.io/serviceaccount/namespace) pod --scanners=misconfig --report all --format json --skip-policy-update | convert_trivy_k8s_to_sarif.py" + + container: + parameters: + # Optional: list of expected return codes, anything else will be considered as an error. by default: [0] + validReturns: [ 0 ] + +kind: ConfigMap +metadata: + name: rapidast-trivy diff --git a/e2e-tests/manifests/rapidast-trivy-pod.yaml b/e2e-tests/manifests/rapidast-trivy-pod.yaml new file mode 100644 index 00000000..e82f4fdf --- /dev/null +++ b/e2e-tests/manifests/rapidast-trivy-pod.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-trivy +spec: + containers: + - command: + - bash + - -c + - ./rapidast.py + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-trivy + resources: + limits: + cpu: 500m + memory: 1Gi + requests: + cpu: 125m + memory: 256Mi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + serviceAccountName: ${SERVICEACCOUNT} # trivy needs perms to query API server + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-trivy + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never diff --git a/e2e-tests/manifests/rapidast-vapi-configmap.yaml b/e2e-tests/manifests/rapidast-vapi-configmap.yaml new file mode 100644 index 00000000..d5e0b9a2 --- /dev/null +++ b/e2e-tests/manifests/rapidast-vapi-configmap.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +data: + config.yaml: |+ + config: + configVersion: 5 + + # `application` contains data related to the application, not to the scans. + application: + shortName: "v5-none-release-test" + url: "http://vapi:5000" + + scanners: + zap: + # define a scan through the ZAP scanner + apiScan: + apis: + apiUrl: "http://vapi:5000/docs/openapi.json" + + passiveScan: + # optional list of passive rules to disable + disabledRules: "2,10015,10027,10096,10024,10098,10023,10105" + + activeScan: + policy: API-scan-minimal + + container: + parameters: + executable: "zap.sh" + + miscOptions: + # enableUI (default: false), requires a compatible runtime (e.g.: flatpak or no containment) + #enableUI: True + # Defaults to True, set False to prevent auto update of ZAP plugins + updateAddons: False + # additionalAddons: ascanrulesBeta + # If set to True and authentication is oauth2_rtoken and api.apiUrl is set, download the API outside of ZAP + oauth2OpenapiManualDownload: False + +kind: ConfigMap +metadata: + name: rapidast-vapi diff --git a/e2e-tests/manifests/rapidast-vapi-pod.yaml b/e2e-tests/manifests/rapidast-vapi-pod.yaml new file mode 100644 index 00000000..a5d7a957 --- /dev/null +++ b/e2e-tests/manifests/rapidast-vapi-pod.yaml @@ -0,0 +1,52 @@ +apiVersion: v1 +kind: Pod +metadata: + annotations: + name: rapidast-vapi +spec: + containers: + - command: + - bash + - -c + - "./rapidast.py && cat results/*/*/zap/zap-report.json" # ugly, but saves needing a PVC to retrieve .json file after execution + # - "./rapidast.py --log-level debug && sleep infinity" # keep the pod alive so we can exec into it to parse reports + env: + - name: HOME + value: /opt/rapidast + image: ${IMAGE} # quay.io/redhatproductsecurity/rapidast:latest + imagePullPolicy: Always + name: rapidast-vapi + resources: + limits: + cpu: 1 + memory: 2Gi + requests: + cpu: 250m + memory: 512Mi + volumeMounts: + - name: config-volume + mountPath: /opt/rapidast/config + # - name: results-volume + # mountPath: /opt/rapidast/results + securityContext: + supplementalGroups: [1000] # "dast" group, necessary to write to /opt/rapidast/results if no PVC mounted + volumes: + - name: config-volume + configMap: + name: rapidast-vapi + # - name: results-volume + # persistentVolumeClaim: + # claimName: rapidast-results + restartPolicy: Never +# --- +# apiVersion: v1 +# kind: PersistentVolumeClaim +# metadata: +# name: rapidast-results +# spec: +# accessModes: +# - ReadWriteOnce +# volumeMode: Filesystem +# resources: +# requests: +# storage: 1Gi diff --git a/e2e-tests/manifests/task-controller-deployment.yaml b/e2e-tests/manifests/task-controller-deployment.yaml new file mode 100644 index 00000000..dc8e4266 --- /dev/null +++ b/e2e-tests/manifests/task-controller-deployment.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + name: task-controller + labels: + app: task-controller +spec: + replicas: 1 + selector: + matchLabels: + app: task-controller + template: + metadata: + labels: + app: task-controller + spec: + containers: + # simulates a custom controller that monitors for tekton Tasks named "vulnerable" + # and tries to execute a deliberate command injection + # Tasks are chosen only because they have a field in .spec that can be arbitrary string + - command: + - bash + - -c + - | + while true; do + sleep 2 + sh -c "$(oc get task/vulnerable -o=jsonpath='{.spec.description}')" + done + image: registry.redhat.io/openshift4/ose-cli:latest + imagePullPolicy: Always + name: task-controller + serviceAccountName: ${SERVICEACCOUNT} # required to read Tasks from API server diff --git a/e2e-tests/manifests/vapi-deployment.yaml b/e2e-tests/manifests/vapi-deployment.yaml new file mode 100644 index 00000000..7b41f204 --- /dev/null +++ b/e2e-tests/manifests/vapi-deployment.yaml @@ -0,0 +1,29 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + name: vapi + labels: + app: vapi +spec: + replicas: 1 + selector: + matchLabels: + app: vapi + template: + metadata: + labels: + app: vapi + spec: + containers: + # command is custom because: + # - default entrypoint runs frontend and based on rapidast-consolidated-e2e.sh we only want backend + # - running start_backend.sh on its own is not sufficient, a sleep is needed to keep the pod alive + - command: + - bash + - -c + - . start_backend.sh && sleep infinity + image: quay.io/sfowler/vapi:latest + imagePullPolicy: Always + name: vapi diff --git a/e2e-tests/manifests/vapi-service.yaml b/e2e-tests/manifests/vapi-service.yaml new file mode 100644 index 00000000..fe64da8b --- /dev/null +++ b/e2e-tests/manifests/vapi-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: vapi + name: vapi +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - port: 5000 + protocol: TCP + targetPort: 5000 + selector: + app: vapi + sessionAffinity: None + type: ClusterIP diff --git a/e2e-tests/test_integration.py b/e2e-tests/test_integration.py new file mode 100644 index 00000000..9e271002 --- /dev/null +++ b/e2e-tests/test_integration.py @@ -0,0 +1,200 @@ +import json +import os +import re +import shutil +import tempfile + +import certifi +import pytest +from kubernetes import client +from kubernetes import config +from kubernetes import utils +from kubernetes import watch +from kubernetes.client.rest import ApiException + +NAMESPACE = os.getenv("RAPIDAST_NAMESPACE", "") # e.g. rapidast--pipeline +SERVICEACCOUNT = os.getenv("RAPIDAST_SERVICEACCOUNT", "pipeline") # name of ServiceAccount used in rapidast pod +RAPIDAST_IMAGE = os.getenv("RAPIDAST_IMAGE", "quay.io/redhatproductsecurity/rapidast:latest") +# delete resources created by tests +RAPIDAST_CLEANUP = os.getenv("RAPIDAST_CLEANUP", "True").lower() in ("true", "1", "t", "y", "yes") + +MANIFESTS = "e2e-tests/manifests" + + +# monkeypatch certifi so that internal CAs are trusted +def where(): + return os.getenv("REQUESTS_CA_BUNDLE", "/etc/pki/tls/certs/ca-bundle.crt") + + +certifi.where = where + + +@pytest.fixture(name="kclient") +def fixture_kclient(): + config.load_config() + yield client.ApiClient() + + +def wait_until_ready(**kwargs): + corev1 = client.CoreV1Api() + w = watch.Watch() + for event in w.stream(func=corev1.list_namespaced_pod, namespace=NAMESPACE, timeout_seconds=60, **kwargs): + if not isinstance(event, dict): # Watch.stream() can yield non-dict types + continue + print(event["object"].metadata.name, event["object"].status.phase) + if event["object"].status.phase == "Running": + return + raise RuntimeError("Timeout out waiting for pod matching: {kwargs}") + + +# simulates: $ oc logs -f | tee +def tee_log(pod_name: str, filename: str): + corev1 = client.CoreV1Api() + w = watch.Watch() + with open(filename, "w", encoding="utf-8") as f: + for e in w.stream(corev1.read_namespaced_pod_log, name=pod_name, namespace=NAMESPACE): + if not isinstance(e, str): + continue # Watch.stream() can yield non-string types + f.write(e + "\n") + print(e) + + +def render_manifests(input_dir, output_dir): + shutil.copytree(input_dir, output_dir, dirs_exist_ok=True) + print(f"rendering manifests in {output_dir}") + # XXX should probably replace this with something like kustomize + for filepath in os.scandir(output_dir): + with open(filepath, "r", encoding="utf-8") as f: + contents = f.read() + contents = contents.replace("${IMAGE}", RAPIDAST_IMAGE) + contents = contents.replace("${SERVICEACCOUNT}", SERVICEACCOUNT) + with open(filepath, "w", encoding="utf-8") as f: + f.write(contents) + + +def setup_namespace(): + global NAMESPACE + # only try to create a namespace if env is set + if NAMESPACE == "": + NAMESPACE = get_current_namespace() + else: + create_namespace(NAMESPACE) + print(f"using namespace '{NAMESPACE}'") + + +def get_current_namespace() -> str: + try: + # Load the kubeconfig + config.load_config() + + # Get the kube config object + _, active_context = config.list_kube_config_contexts() + + # Return the namespace from current context + if active_context and "namespace" in active_context["context"]: + return active_context["context"]["namespace"] + return "default" + + except config.config_exception.ConfigException: + # If running inside a pod + try: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r") as f: + return f.read().strip() + except FileNotFoundError: + return "default" + + +def create_namespace(namespace_name: str): + config.load_config() + corev1 = client.CoreV1Api() + try: + corev1.read_namespace(namespace_name) + print(f"namespace {namespace_name} already exists") + except ApiException as e: + if e.status == 404: + print(f"creating namespace {namespace_name}") + namespace = client.V1Namespace(metadata=client.V1ObjectMeta(name=namespace_name)) + corev1.create_namespace(namespace) + else: + raise e + except Exception as e: + print(f"error reading namesapce {namespace_name}: {e}") + + +def cleanup(): + if RAPIDAST_CLEANUP: + os.system(f"kubectl delete -f {MANIFESTS}/") + # XXX oobtukbe does not clean up after itself + os.system("kubectl delete Task/vulnerable") + + +class TestRapiDAST: + @classmethod + def setup_class(cls): + cls.tempdir = tempfile.mkdtemp() + render_manifests(MANIFESTS, cls.tempdir) + print(f"testing with image: {RAPIDAST_IMAGE}") + setup_namespace() + cleanup() + + @classmethod + def teardown_class(cls): + # TODO teardown should really occur after each test, so the the + # resource count does not grown until quota reached + cleanup() + + def create_from_yaml(self, kclient, path: str): + # simple wrapper to reduce repetition + utils.create_from_yaml(kclient, path, namespace=NAMESPACE, verbose=True) + + def test_vapi(self, kclient): + """Test rapidast find expected number of findings in VAPI""" + self.create_from_yaml(kclient, f"{self.tempdir}/vapi-deployment.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/vapi-service.yaml") + wait_until_ready(label_selector="app=vapi") + + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-vapi-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-vapi-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-vapi") + + logfile = os.path.join(self.tempdir, "rapidast-vapi.log") + tee_log("rapidast-vapi", logfile) + + # XXX relies on rapidast-vapi pod cat-ing the result json file after execution + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + pattern = r"^{\s*$.*$" + matches = re.findall(pattern, logs, re.MULTILINE | re.DOTALL) + assert matches, f"{logfile} did not contain expected json results" + results = json.loads(matches[0]) + + assert len(results["site"][0]["alerts"]) == 3 + + def test_trivy(self, kclient): + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-trivy-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-trivy-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-trivy") + + logfile = os.path.join(self.tempdir, "rapidast-trivy.log") + tee_log("rapidast-trivy", logfile) + + expected_line = "INFO:scanner: 'generic_trivy' completed successfully" + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + assert expected_line in logs, f"{logfile} does not contain expected line: {expected_line}" + + def test_oobtkube(self, kclient): + self.create_from_yaml(kclient, f"{self.tempdir}/task-controller-deployment.yaml") + + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-configmap.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-service.yaml") + self.create_from_yaml(kclient, f"{self.tempdir}/rapidast-oobtkube-pod.yaml") + wait_until_ready(field_selector="metadata.name=rapidast-oobtkube") + + logfile = os.path.join(self.tempdir, "rapidast-oobtkube.log") + tee_log("rapidast-oobtkube", logfile) + + expected_line = "RESULT: OOB REQUEST DETECTED" + with open(logfile, "r", encoding="utf-8") as f: + logs = f.read() + assert expected_line in logs, f"{logfile} does not contain expected line: {expected_line}" diff --git a/requirements-dev.txt b/requirements-dev.txt index 3993dc3e..20e11a9f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,3 +4,5 @@ pytest >= 7.2.1 black requests pre-commit == 3.7.1 +kubernetes +pytest-json-report From 5d00309f8e9ffb9c542a1eb7b29517d010c5440f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 29 Oct 2024 16:58:58 +1000 Subject: [PATCH 03/10] Skip slow checks --- .tekton/integration-test.yaml | 156 +++++++++++-- .tekton/rapidast-pull-request.yaml | 4 +- .../manifests/integration-test-pipeline.yaml | 211 ------------------ 3 files changed, 139 insertions(+), 232 deletions(-) delete mode 100644 e2e-tests/manifests/integration-test-pipeline.yaml diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml index 9af4aab1..9a537e46 100644 --- a/.tekton/integration-test.yaml +++ b/.tekton/integration-test.yaml @@ -1,23 +1,139 @@ -apiVersion: appstudio.redhat.com/v1beta2 -kind: IntegrationTestScenario +--- +apiVersion: tekton.dev/v1beta1 +kind: Pipeline metadata: - name: experiment-test - annotations: - pipelinesascode.tekton.dev/on-event: "[pull_request, push]" - pipelinesascode.tekton.dev/on-target-branch: "[main, development]" - labels: - test.appstudio.openshift.io/optional: "true" + name: rapidast-e2e spec: - application: rapidast - contexts: - - description: Application testing - name: application - resolverRef: - params: - - name: url - value: https://github.com/sfowl/rapidast + params: + - name: repo_url + default: github.com/sfowl/rapidast - name: revision - value: "{{ revision }}" - - name: pathInRepo - value: e2e-tests/manifests/integration-test-pipeline.yaml - resolver: git + default: "development" + - name: image + default: quay.io/redhatproductsecurity/rapidast:latest + - description: 'Snapshot of the application' + name: SNAPSHOT + default: |- + '{ + "components": [ + { + "name":"rapidast", + "containerImage": "quay.io/redhatproductsecurity/rapidast:latest", + "source":{ + "git":{ + "url":"git@github.com:sfowl/rapidast.git", + "revision":"development", + } + } + } + ] + }' + type: string + + tasks: + - name: parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/integration-examples + - name: revision + value: main + - name: pathInRepo + value: tasks/test_metadata.yaml + params: + - name: SNAPSHOT + value: $(params.SNAPSHOT) + + - name: provision-eaas-space + runAfter: + - parse-metadata + taskRef: + resolver: git + params: + - name: url + value: https://github.com/konflux-ci/build-definitions.git + - name: revision + value: main + - name: pathInRepo + value: task/provision-env-with-ephemeral-namespace/0.1/provision-env-with-ephemeral-namespace.yaml + params: + - name: KONFLUXNAMESPACE + value: $(context.pipelineRun.namespace) + - name: PIPELINERUN_NAME + value: $(context.pipelineRun.name) + - name: PIPELINERUN_UID + value: $(context.pipelineRun.uid) + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + # + # - name: clone-repository + # runAfter: + # - parse-metadata + # params: + # - name: url + # value: "$(tasks.parse-metadata.results.source-git-url)" + # - name: revision + # value: "$(tasks.parse-metadata.results.source-git-revision)" + # taskRef: + # name: git-clone + # workspaces: + # - name: output + # workspace: source + + - name: run-e2e-tests + runAfter: + - provision-eaas-space + taskSpec: + volumes: + - name: credentials + emptyDir: {} + results: + - name: TEST_RESULTS + description: e2e test results + steps: + + # XXX not supported to use workspaces in integration tests: + # * https://issues.redhat.com/browse/STONEINTG-895 + - name: clone-repository + image: quay.io/konflux-ci/git-clone:latest + script: | + git config --global --add safe.directory /workspace + git clone "$(tasks.parse-metadata.results.source-git-url)" /workspace + pushd /workspace + git checkout "$(tasks.parse-metadata.results.source-git-revision)" + # XXX Temporary hardcoded repo + commit ! + # git checkout test-e2e-test + + - name: test + image: registry.redhat.io/openshift4/ose-cli:latest + env: + - name: KUBECONFIG + value: /tmp/kubeconfig + - name: KUBECONFIG_VALUE + valueFrom: + secretKeyRef: + name: $(tasks.provision-eaas-space.results.secretRef) + key: kubeconfig + - name: RAPIDAST_CLEANUP + value: "false" # namespace will be cleaned up automatically + - name: RAPIDAST_IMAGE + value: $(tasks.parse-metadata.results.component-container-image) + - name: RAPIDAST_SERVICEACCOUNT + value: namespace-manager # created by provision-env-with-ephemeral-namespace + workingDir: /workspace + volumeMounts: + - name: credentials + mountPath: /credentials + script: | + #!/bin/bash -ex + + echo "$KUBECONFIG_VALUE" > "$KUBECONFIG" + oc whoami + + yum install -y python3.12 + python3.12 -m ensurepip + pip3 install -r requirements.txt -r requirements-dev.txt + pytest -s e2e-tests --json-report --json-report-summary --json-report-file $(results.TEST_RESULTS.path) + cat $(results.TEST_RESULTS.path) diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml index 9d774d10..2acfe3fe 100644 --- a/.tekton/rapidast-pull-request.yaml +++ b/.tekton/rapidast-pull-request.yaml @@ -25,9 +25,11 @@ spec: - name: output-image value: quay.io/redhat-user-workloads/secaut-tenant/rapidast:on-pr-{{revision}} - name: image-expires-after - value: 5d + value: 7d - name: dockerfile value: containerize/Containerfile + - name: skip-checks + value: "true" pipelineSpec: description: | This pipeline is ideal for building container images from a Containerfile while maintaining trust after pipeline customization. diff --git a/e2e-tests/manifests/integration-test-pipeline.yaml b/e2e-tests/manifests/integration-test-pipeline.yaml deleted file mode 100644 index 121be3fe..00000000 --- a/e2e-tests/manifests/integration-test-pipeline.yaml +++ /dev/null @@ -1,211 +0,0 @@ ---- -apiVersion: tekton.dev/v1beta1 -kind: Pipeline -metadata: - name: rapidast-e2e -spec: - params: - - name: repo_url - default: github.com/sfowl/rapidast - - name: revision - default: "development" - - name: image - default: quay.io/redhatproductsecurity/rapidast:latest - - description: 'Snapshot of the application' - name: SNAPSHOT - default: |- - '{ - "components": [ - { - "name":"rapidast", - "containerImage": "quay.io/redhatproductsecurity/rapidast:latest", - "source":{ - "git":{ - "url":"git@github.com:sfowl/rapidast.git", - "revision":"development", - } - } - } - ] - }' - type: string - - tasks: - - name: parse-metadata - taskRef: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/integration-examples - - name: revision - value: main - - name: pathInRepo - value: tasks/test_metadata.yaml - params: - - name: SNAPSHOT - value: $(params.SNAPSHOT) - - - name: provision-eaas-space - runAfter: - - parse-metadata - taskRef: - resolver: git - params: - - name: url - value: https://github.com/konflux-ci/build-definitions.git - - name: revision - value: main - - name: pathInRepo - value: task/provision-env-with-ephemeral-namespace/0.1/provision-env-with-ephemeral-namespace.yaml - params: - - name: KONFLUXNAMESPACE - value: $(context.pipelineRun.namespace) - - name: PIPELINERUN_NAME - value: $(context.pipelineRun.name) - - name: PIPELINERUN_UID - value: $(context.pipelineRun.uid) - - # - name: provision-cluster - # runAfter: - # - provision-eaas-space - # taskSpec: - # results: - # - name: clusterName - # value: "$(steps.create-cluster.results.clusterName)" - # volumes: - # - name: credentials - # emptyDir: {} - # steps: - # - name: get-supported-versions - # ref: - # resolver: git - # params: - # - name: url - # value: https://github.com/konflux-ci/build-definitions.git - # - name: revision - # value: main - # - name: pathInRepo - # value: stepactions/eaas-get-supported-ephemeral-cluster-versions/0.1/eaas-get-supported-ephemeral-cluster-versions.yaml - # params: - # - name: eaasSpaceSecretRef - # value: $(tasks.provision-eaas-space.results.secretRef) - # - name: pick-version - # ref: - # resolver: git - # params: - # - name: url - # value: https://github.com/konflux-ci/build-definitions.git - # - name: revision - # value: main - # - name: pathInRepo - # value: stepactions/eaas-get-latest-openshift-version-by-prefix/0.1/eaas-get-latest-openshift-version-by-prefix.yaml - # params: - # - name: prefix - # value: "$(steps.get-supported-versions.results.versions[0])." - # - name: create-cluster - # ref: - # resolver: git - # params: - # - name: url - # value: https://github.com/konflux-ci/build-definitions.git - # - name: revision - # value: main - # - name: pathInRepo - # value: stepactions/eaas-create-ephemeral-cluster-hypershift-aws/0.1/eaas-create-ephemeral-cluster-hypershift-aws.yaml - # params: - # - name: eaasSpaceSecretRef - # value: $(tasks.provision-eaas-space.results.secretRef) - # - name: version - # value: "$(steps.pick-version.results.version)" - - # XXX not supported to use workspaces in integration tests: - # * https://issues.redhat.com/browse/STONEINTG-895 - # - # - name: clone-repository - # runAfter: - # - parse-metadata - # params: - # - name: url - # value: "$(tasks.parse-metadata.results.source-git-url)" - # - name: revision - # value: "$(tasks.parse-metadata.results.source-git-revision)" - # taskRef: - # name: git-clone - # workspaces: - # - name: output - # workspace: source - - - name: run-e2e-tests - runAfter: - - provision-eaas-space - taskSpec: - volumes: - - name: credentials - emptyDir: {} - results: - - name: TEST_RESULTS - description: e2e test results - steps: - # - name: get-kubeconfig - # ref: - # resolver: git - # params: - # - name: url - # value: https://github.com/konflux-ci/build-definitions.git - # - name: revision - # value: main - # - name: pathInRepo - # value: stepactions/eaas-get-ephemeral-cluster-credentials/0.1/eaas-get-ephemeral-cluster-credentials.yaml - # params: - # - name: eaasSpaceSecretRef - # value: $(tasks.provision-eaas-space.results.secretRef) - # - name: clusterName - # value: "$(tasks.provision-cluster.results.clusterName)" - # - name: credentials - # value: credentials - - - # XXX not supported to use workspaces in integration tests: - # * https://issues.redhat.com/browse/STONEINTG-895 - - name: clone-repository - image: quay.io/konflux-ci/git-clone:latest - script: | - git config --global --add safe.directory /workspace - git clone "$(tasks.parse-metadata.results.source-git-url)" /workspace - pushd /workspace - # git checkout "$(tasks.parse-metadata.results.source-git-revision)" - # XXX Temporary hardcoded repo + commit ! - git checkout test-e2e-test - - - name: test - image: registry.redhat.io/openshift4/ose-cli:latest - env: - - name: KUBECONFIG - value: /tmp/kubeconfig - # value: "/credentials/$(steps.get-kubeconfig.results.kubeconfig)" - - name: KUBECONFIG_VALUE - valueFrom: - secretKeyRef: - name: $(tasks.provision-eaas-space.results.secretRef) - key: kubeconfig - - name: RAPIDAST_CLEANUP - value: "false" # namespace will be cleanedup automatically - - name: RAPIDAST_IMAGE - value: $(tasks.parse-metadata.results.component-container-image) - - name: RAPIDAST_SERVICEACCOUNT - value: namespace-manager # created by provision-env-with-ephemeral-namespace - workingDir: /workspace - volumeMounts: - - name: credentials - mountPath: /credentials - script: | - #!/bin/bash -ex - - echo "$KUBECONFIG_VALUE" > "$KUBECONFIG" - oc whoami - - yum install -y python3.12 - python3.12 -m ensurepip - pip3 install -r requirements.txt -r requirements-dev.txt - pytest -s e2e-tests --json-report --json-report-summary --json-report-file $(results.TEST_RESULTS.path) - cat $(results.TEST_RESULTS.path) From 102e59bbca863cda9100d9ce5bce680b585520f6 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 4 Nov 2024 15:23:37 +1000 Subject: [PATCH 04/10] fixup pylint errors --- .tekton/integration-test.yaml | 2 +- e2e-tests/test_integration.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml index 9a537e46..0fdaab9b 100644 --- a/.tekton/integration-test.yaml +++ b/.tekton/integration-test.yaml @@ -128,7 +128,7 @@ spec: mountPath: /credentials script: | #!/bin/bash -ex - + echo "$KUBECONFIG_VALUE" > "$KUBECONFIG" oc whoami diff --git a/e2e-tests/test_integration.py b/e2e-tests/test_integration.py index 9e271002..2b2fa4f4 100644 --- a/e2e-tests/test_integration.py +++ b/e2e-tests/test_integration.py @@ -73,7 +73,7 @@ def render_manifests(input_dir, output_dir): def setup_namespace(): - global NAMESPACE + global NAMESPACE # pylint: disable=W0603 # only try to create a namespace if env is set if NAMESPACE == "": NAMESPACE = get_current_namespace() @@ -98,7 +98,7 @@ def get_current_namespace() -> str: except config.config_exception.ConfigException: # If running inside a pod try: - with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r") as f: + with open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", "r", encoding="utf-8") as f: return f.read().strip() except FileNotFoundError: return "default" @@ -117,7 +117,7 @@ def create_namespace(namespace_name: str): corev1.create_namespace(namespace) else: raise e - except Exception as e: + except Exception as e: # pylint: disable=W0718 print(f"error reading namesapce {namespace_name}: {e}") From b1780816527e4f96298d2ae5d50be66ea37e5961 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 4 Nov 2024 16:19:01 +1000 Subject: [PATCH 05/10] push to custom repo --- .tekton/integration-test.yaml | 2 -- .tekton/rapidast-pull-request.yaml | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml index 0fdaab9b..052820b8 100644 --- a/.tekton/integration-test.yaml +++ b/.tekton/integration-test.yaml @@ -9,8 +9,6 @@ spec: default: github.com/sfowl/rapidast - name: revision default: "development" - - name: image - default: quay.io/redhatproductsecurity/rapidast:latest - description: 'Snapshot of the application' name: SNAPSHOT default: |- diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml index 2acfe3fe..64e557d6 100644 --- a/.tekton/rapidast-pull-request.yaml +++ b/.tekton/rapidast-pull-request.yaml @@ -23,7 +23,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/secaut-tenant/rapidast:on-pr-{{revision}} + value: quay.io/sfowler/rapidast:on-pr-{{revision}} - name: image-expires-after value: 7d - name: dockerfile From c402ae4c9f2566ea100ee0282ca5a8e1bee17616 Mon Sep 17 00:00:00 2001 From: Sam Date: Mon, 4 Nov 2024 17:37:50 +1000 Subject: [PATCH 06/10] Remove github actions --- .github/workflows/build-and-push.yml | 41 ---------------------------- .github/workflows/build-image.yml | 26 ------------------ .github/workflows/run-tests.yml | 21 +++++++++++--- .tekton/rapidast-pull-request.yaml | 3 +- 4 files changed, 18 insertions(+), 73 deletions(-) delete mode 100644 .github/workflows/build-and-push.yml delete mode 100644 .github/workflows/build-image.yml diff --git a/.github/workflows/build-and-push.yml b/.github/workflows/build-and-push.yml deleted file mode 100644 index f1974b03..00000000 --- a/.github/workflows/build-and-push.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Build and push container image - -env: - IMAGE_NAME: "rapidast" - IMAGE_TAGS: "${{ github.sha }}" - IMAGE_REGISTRY: quay.io/redhatproductsecurity - IMAGE_REGISTRY_USER: ${{ secrets.IMAGE_REGISTRY_USER }} - IMAGE_REGISTRY_PASSWORD: ${{ secrets.IMAGE_REGISTRY_PASSWORD }} - -on: - push: - branches: ["development", "main"] - -jobs: - - build-and-push: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - # https://github.com/redhat-actions/buildah-build#readme - - name: Build container image - id: build-image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - dockerfiles: | - ./containerize/Containerfile - - # https://github.com/redhat-actions/push-to-registry#readme - - name: Push to registry - id: push-image - uses: redhat-actions/push-to-registry@v2 - with: - image: ${{ steps.build-image.outputs.image }} - tags: ${{ steps.build-image.outputs.tags }} - registry: ${{ env.IMAGE_REGISTRY }} - username: ${{ env.IMAGE_REGISTRY_USER }} - password: ${{ env.IMAGE_REGISTRY_PASSWORD }} diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml deleted file mode 100644 index e1ce7f9e..00000000 --- a/.github/workflows/build-image.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Build container image - -env: - IMAGE_NAME: "rapidast" - IMAGE_TAGS: "${{ github.sha }}" - -on: - pull_request: - branches: ["development", "main"] - -jobs: - - build-image: - - runs-on: ubuntu-latest - - # https://github.com/redhat-actions/buildah-build#readme - steps: - - uses: actions/checkout@v4 - - name: Build container image - uses: redhat-actions/buildah-build@v2 - with: - image: ${{ env.IMAGE_NAME }} - tags: ${{ env.IMAGE_TAGS }} - dockerfiles: | - ./containerize/Containerfile diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index b43b9b32..c9f8e8a1 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -10,8 +10,24 @@ permissions: contents: read jobs: - test: + lint: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.9 + uses: actions/setup-python@v3 + with: + python-version: "3.9" + - name: Install dependencies + run: | + python3 -m ensurepip --upgrade + pip install --no-cache-dir -r requirements.txt -r requirements-dev.txt + - name: Lint with pre-commit hook + run: | + pre-commit run --all-files --show-diff-on-failure + unit-tests: runs-on: ubuntu-latest steps: @@ -27,6 +43,3 @@ jobs: - name: Test with pytest run: | pytest tests - - name: Lint with pre-commit hook - run: | - pre-commit run --all-files --show-diff-on-failure diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml index 64e557d6..b0c69dd3 100644 --- a/.tekton/rapidast-pull-request.yaml +++ b/.tekton/rapidast-pull-request.yaml @@ -7,8 +7,7 @@ metadata: build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch - == "development" + pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && (target_branch == "development" || target_branch == "main") creationTimestamp: null labels: appstudio.openshift.io/application: rapidast From b66e04fb14ad4001d61b9c1e068ae5f42395421f Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 5 Nov 2024 11:15:53 +1000 Subject: [PATCH 07/10] Update to redhatproductsecurity repos --- .tekton/integration-test.yaml | 4 ++-- .tekton/rapidast-pull-request.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml index 052820b8..dac19d3b 100644 --- a/.tekton/integration-test.yaml +++ b/.tekton/integration-test.yaml @@ -6,7 +6,7 @@ metadata: spec: params: - name: repo_url - default: github.com/sfowl/rapidast + default: github.com/RedHatProductSecurity/rapidast - name: revision default: "development" - description: 'Snapshot of the application' @@ -19,7 +19,7 @@ spec: "containerImage": "quay.io/redhatproductsecurity/rapidast:latest", "source":{ "git":{ - "url":"git@github.com:sfowl/rapidast.git", + "url":"git@github.com:RedHatProductSecurity/rapidast.git", "revision":"development", } } diff --git a/.tekton/rapidast-pull-request.yaml b/.tekton/rapidast-pull-request.yaml index b0c69dd3..3e833400 100644 --- a/.tekton/rapidast-pull-request.yaml +++ b/.tekton/rapidast-pull-request.yaml @@ -22,7 +22,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/sfowler/rapidast:on-pr-{{revision}} + value: quay.io/redhatproductsecurity/rapidast:on-pr-{{revision}} - name: image-expires-after value: 7d - name: dockerfile From a183919b569ae6dce8b9652ce673a16a3e013ca1 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 5 Nov 2024 13:43:06 +1000 Subject: [PATCH 08/10] fixup image location --- .tekton/rapidast-push.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.tekton/rapidast-push.yaml b/.tekton/rapidast-push.yaml index 22f9131b..759c9e22 100644 --- a/.tekton/rapidast-push.yaml +++ b/.tekton/rapidast-push.yaml @@ -6,8 +6,7 @@ metadata: build.appstudio.redhat.com/commit_sha: '{{revision}}' build.appstudio.redhat.com/target_branch: '{{target_branch}}' pipelinesascode.tekton.dev/max-keep-runs: "3" - pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch - == "development" + pipelinesascode.tekton.dev/on-cel-expression: event == "push" && (target_branch == "development" || target_branch == "main") creationTimestamp: null labels: appstudio.openshift.io/application: rapidast @@ -22,7 +21,7 @@ spec: - name: revision value: '{{revision}}' - name: output-image - value: quay.io/redhat-user-workloads/secaut-tenant/rapidast:{{revision}} + value: quay.io/redhatproductsecurity/rapidast:{{revision}} - name: dockerfile value: containerize/Containerfile pipelineSpec: From 2f46904683fbc60d16edfaeb75ffe34ca9a339e5 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 5 Nov 2024 14:40:06 +1000 Subject: [PATCH 09/10] cleanup --- .tekton/integration-test.yaml | 2 -- e2e-tests/test_integration.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.tekton/integration-test.yaml b/.tekton/integration-test.yaml index dac19d3b..1fcd1d05 100644 --- a/.tekton/integration-test.yaml +++ b/.tekton/integration-test.yaml @@ -101,8 +101,6 @@ spec: git clone "$(tasks.parse-metadata.results.source-git-url)" /workspace pushd /workspace git checkout "$(tasks.parse-metadata.results.source-git-revision)" - # XXX Temporary hardcoded repo + commit ! - # git checkout test-e2e-test - name: test image: registry.redhat.io/openshift4/ose-cli:latest diff --git a/e2e-tests/test_integration.py b/e2e-tests/test_integration.py index 2b2fa4f4..65472b11 100644 --- a/e2e-tests/test_integration.py +++ b/e2e-tests/test_integration.py @@ -62,6 +62,7 @@ def tee_log(pod_name: str, filename: str): def render_manifests(input_dir, output_dir): shutil.copytree(input_dir, output_dir, dirs_exist_ok=True) print(f"rendering manifests in {output_dir}") + print(f"using serviceaccount {SERVICEACCOUNT}") # XXX should probably replace this with something like kustomize for filepath in os.scandir(output_dir): with open(filepath, "r", encoding="utf-8") as f: From 67d07257c8b6f0b3b8d64f7406f2c26f6ec0a6d6 Mon Sep 17 00:00:00 2001 From: Sam Date: Tue, 5 Nov 2024 16:22:40 +1000 Subject: [PATCH 10/10] fix typo --- e2e-tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e-tests/test_integration.py b/e2e-tests/test_integration.py index 65472b11..cd8126a1 100644 --- a/e2e-tests/test_integration.py +++ b/e2e-tests/test_integration.py @@ -119,7 +119,7 @@ def create_namespace(namespace_name: str): else: raise e except Exception as e: # pylint: disable=W0718 - print(f"error reading namesapce {namespace_name}: {e}") + print(f"error reading namespace {namespace_name}: {e}") def cleanup():