From 73ff421f5f0375778a6aa35deecb88621032b118 Mon Sep 17 00:00:00 2001 From: Kostas Stamatakis Date: Thu, 14 Mar 2024 14:04:01 +0200 Subject: [PATCH] k8s --- .github/actions/hermit/action.yml | 12 ++ .github/actions/k8s-ci/action.yml | 117 ++++++++++++++++++ .github/workflows/pr-ci.yml | 50 ++++++++ .../packaging/docker/elastic-agent/build.sh | 2 - 4 files changed, 179 insertions(+), 2 deletions(-) create mode 100644 .github/actions/k8s-ci/action.yml diff --git a/.github/actions/hermit/action.yml b/.github/actions/hermit/action.yml index f2d305a3aa..9d04a9f99b 100644 --- a/.github/actions/hermit/action.yml +++ b/.github/actions/hermit/action.yml @@ -33,6 +33,16 @@ runs: restore-keys: | ci-hermit-env-${{ runner.os }} + - if: ${{ inputs.cache == 'true' }} + id: cache-go-deps + uses: actions/cache@v4 + with: + path: | + ~/go/pkg/ + key: ci-go-deps-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('**/go.mod', '**/go.sum') }} + restore-keys: | + ci-go-deps-${{ runner.os }}-${{ runner.arch }} + - name: Initialize hermit shell: bash run: ./bin/hermit env --raw >> "$GITHUB_ENV" @@ -59,5 +69,7 @@ runs: shell: bash run: pre-commit || true + - shell: bash + run: go mod download - shell: bash run: go env diff --git a/.github/actions/k8s-ci/action.yml b/.github/actions/k8s-ci/action.yml new file mode 100644 index 0000000000..dd97fc3bbc --- /dev/null +++ b/.github/actions/k8s-ci/action.yml @@ -0,0 +1,117 @@ +name: 'K8S CI' +description: 'K8s integration tests' +inputs: + elk-version: + description: 'ELK version' + required: true + kind-config: + description: 'KIND configuration' + required: true + container-image-suffix: + description: 'Container image suffix' + required: true + elastic-agent-docker-image: + description: 'Elastic-Agent docker image' + required: true + elastic-agent-docker-image-tag: + description: 'Elastic-Agent docker image tag' + required: true + + test-target: + description: 'Test target' + required: true + test-range: + description: 'Test range' + required: false + default: '' + values-file: + description: 'Helm values file' + required: true +runs: + using: composite + steps: + - name: Free Disk Space (Ubuntu) + uses: jlumbroso/free-disk-space@main + with: + tool-cache: false + android: true + dotnet: true + haskell: true + large-packages: false + docker-images: true + swap-storage: true + + - name: Build cloudbeat binary + shell: bash + run: mage -v build + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build cloudbeat-docker image + uses: docker/build-push-action@v5 + with: + context: . + file: ./deploy/Dockerfile + push: false + tags: cloudbeat:latest + #cache-from: type=local,src=/tmp/.buildx-cache + #cache-to: type=local,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/cloudbeat-${{ inputs.container-image-suffix }}.tar + + - name: Build elastic-agent + env: + BASE_IMAGE: docker.elastic.co/beats/elastic-agent:${{ inputs.elastic-agent-docker-image-tag }} + GOOS: linux + GOARCH: amd64 + shell: bash + run: ./scripts/packaging/docker/elastic-agent/build.sh -t ${{ inputs.elastic-agent-docker-image }}:${{ inputs.elastic-agent-docker-image-tag }} + + - name: Build pytest-docker + uses: docker/build-push-action@v5 + with: + context: ./tests/. + push: false + tags: cloudbeat-test:latest + #cache-from: type=local,mode=max,src=/tmp/.buildx-cache + #cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new + outputs: type=docker,dest=/tmp/pytest-${{ inputs.container-image-suffix }}.tar + + - name: Prepare Kind Cluster for Process Tests + if: ${{ contains(inputs.kind-config, 'conf2') }} + shell: bash + run: | + # Workaround: Direct creation of a Kind cluster for config2 fails to start control-plane (kubelet fails) + # Creating and deleting a kind-mono cluster as a preparation for configuring Kind for process tests + just create-kind-cluster kind-mono + just delete-kind-cluster kind-mono + + - name: Create k8s Kind Cluster + shell: bash + run: just create-kind-cluster ${{ inputs.kind-config }} + + - name: Load images to kind + shell: bash + run: ./.ci/scripts/kind-images.sh '${{ inputs.container-image-suffix }}' '${{ inputs.kind-config }}' + + - name: Deploy tests Helm chart + id: deploy_helm + shell: bash + run: just deploy-tests-helm '${{ inputs.test-target }}' '${{ inputs.values-file }}' '${{ inputs.test-range }}' + + - name: Deploy K8s Objects + if: ${{ contains(inputs.test-target, 'object_psp') }} + shell: bash + run: just apply-k8s-test-objects + + - name: Run Tests + id: run_tests + shell: bash + run: just run-tests ${{ inputs.test-target }} kind-${{ inputs.kind-config }} + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: allure-results-ci-k8s-${{ inputs.test-target }}-${{ inputs.kind-config }} + path: tests/allure/results/ diff --git a/.github/workflows/pr-ci.yml b/.github/workflows/pr-ci.yml index 7c720b90d6..95fff8aaa1 100644 --- a/.github/workflows/pr-ci.yml +++ b/.github/workflows/pr-ci.yml @@ -113,12 +113,62 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: 'us-east-2' + ci-k8s: + needs: [ init-hermit ] + name: ${{ matrix.test-target }}-${{ matrix.kind-config }} + runs-on: ubuntu-22.04 + timeout-minutes: 120 + strategy: + fail-fast: false + matrix: + include: + - test-target: pre_merge + kind-config: kind-multi + values-file: tests/test_environments/values/ci.yml + - test-target: pre_merge_agent + kind-config: kind-multi + values-file: tests/test_environments/values/ci-sa-agent.yml + - test-target: k8s_file_system_rules + kind-config: kind-test-files + values-file: tests/test_environments/values/ci-test-k8s-files.yml + - test-target: k8s_object_psp_rules + kind-config: kind-multi + values-file: tests/test_environments/values/ci-test-k8s-objects.yml + - test-target: k8s_process_rules + kind-config: kind-test-proc-conf1 + values-file: tests/test_environments/values/ci-test-k8s-proc-conf1.yml + - test-target: k8s_process_rules + kind-config: kind-test-proc-conf2 + values-file: tests/test_environments/values/ci-test-k8s-proc-conf2.yml + env: + CONTAINER_SUFFIX: ${{ github.run_id }} + CI_ELASTIC_AGENT_DOCKER_TAG: "8.14.0-SNAPSHOT" + CI_ELASTIC_AGENT_DOCKER_IMAGE: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent" + steps: + - name: Check out the repo + uses: actions/checkout@v4 + + - name: Hermit Environment + uses: ./.github/actions/hermit + + - name: Run k8s integration tests + uses: ./.github/actions/k8s-ci + with: + elk-version: ${{ env.ELK_VERSION }} + kind-config: ${{ matrix.kind-config }} + container-image-suffix: ${{ github.run_id }} + elastic-agent-docker-image: ${{ env.CI_ELASTIC_AGENT_DOCKER_IMAGE }} + elastic-agent-docker-image-tag: ${{ env.CI_ELASTIC_AGENT_DOCKER_TAG }} + test-target: ${{ matrix.test-target }} + values-file: ${{ matrix.values-file }} + upload-allure-results: needs: - ci-azure - ci-aws - ci-gcp - ci-cnvm + - ci-k8s name: Upload integration tests results runs-on: ubuntu-22.04 timeout-minutes: 60 diff --git a/scripts/packaging/docker/elastic-agent/build.sh b/scripts/packaging/docker/elastic-agent/build.sh index 31bd1ad525..e76dd30485 100755 --- a/scripts/packaging/docker/elastic-agent/build.sh +++ b/scripts/packaging/docker/elastic-agent/build.sh @@ -24,7 +24,5 @@ docker buildx build \ --build-arg STACK_VERSION=$STACK_VERSION \ --build-arg VCS_REF_SHORT=${VCS_REF:0:6} \ --platform linux/$GOARCH \ - --cache-from=type=local,src=/tmp/.buildx-cache \ - --cache-to=type=local,dest=/tmp/.buildx-cache-new \ --output type=docker,dest=/tmp/elastic-agent-$CONTAINER_SUFFIX.tar \ $* $REPO_ROOT