Daily e2e tests #528
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # THIS FILE IS GENERATED, PLEASE DO NOT EDIT. | |
| # | |
| # Copyright 2022 Flant JSC | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| name: 'Daily e2e tests' | |
| on: | |
| schedule: | |
| - cron: '0 1 * * 1-5' | |
| workflow_dispatch: | |
| env: | |
| WERF_DRY_RUN: "false" | |
| # <template: werf_envs> | |
| WERF_CHANNEL: "ea" | |
| WERF_ENV: "FE" | |
| TEST_TIMEOUT: "15m" | |
| # Use fixed string 'sys/deckhouse-oss' for repo name. ${CI_PROJECT_PATH} is not available here in GitHub. | |
| DEV_REGISTRY_PATH: "${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/sys/deckhouse-oss" | |
| # Registry for additional repositories used for testing Github Actions workflows. | |
| GHA_TEST_REGISTRY_PATH: "ghcr.io/${{ github.repository }}" | |
| # </template: werf_envs> | |
| # Always run a single job at a time. | |
| # Note: Concurrency is currently in beta and subject to change. | |
| # https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency | |
| concurrency: | |
| group: e2e-daily | |
| jobs: | |
| skip_tests_repos: | |
| name: Skip tests repos | |
| runs-on: ubuntu-latest | |
| if: ${{ false || github.repository == 'deckhouse/deckhouse' }} | |
| steps: | |
| - name: Do nothing | |
| run: echo "Empty action to fulfil Github requirements." | |
| # Note: git_info is needed for werf.yaml | |
| # <template: git_info_job> | |
| git_info: | |
| name: Get git info | |
| runs-on: ubuntu-latest | |
| needs: | |
| - skip_tests_repos | |
| outputs: | |
| ci_commit_tag: ${{ steps.git_info.outputs.ci_commit_tag }} | |
| ci_commit_branch: ${{ steps.git_info.outputs.ci_commit_branch }} | |
| ci_commit_ref_name: ${{ steps.git_info.outputs.ci_commit_ref_name }} | |
| ci_commit_ref_slug: ${{ steps.git_info.outputs.ci_commit_ref_slug }} | |
| ref_full: ${{ steps.git_info.outputs.ref_full }} | |
| github_sha: ${{ steps.git_info.outputs.github_sha }} | |
| pr_number: ${{ steps.git_info.outputs.pr_number }} | |
| # Skip the CI for automation PRs, e.g. changelog | |
| if: ${{ github.event.pull_request.user.login != 'deckhouse-BOaTswain' }} | |
| steps: | |
| - id: git_info | |
| name: Get tag name and SHA | |
| uses: actions/[email protected] | |
| with: | |
| script: | | |
| const { GITHUB_REF_TYPE, GITHUB_REF_NAME, GITHUB_REF } = process.env | |
| let refSlug = '' | |
| let refName = '' | |
| let refFull = '' | |
| let githubBranch = '' | |
| let githubTag = '' | |
| let githubSHA = '' | |
| let prNumber = '' | |
| if (context.eventName === "workflow_dispatch" && context.payload.inputs && context.payload.inputs.pull_request_ref) { | |
| // Trigger: workflow_dispatch with pull_request_ref. | |
| // Extract pull request number from 'refs/pull/<NUM>/merge' | |
| prNumber = context.payload.inputs.pull_request_ref.replace('refs/pull/', '').replace('/merge', '').replace('/head', '') | |
| refSlug = `pr${prNumber}` | |
| refName = context.payload.inputs.ci_commit_ref_name | |
| refFull = context.payload.inputs.pull_request_ref | |
| githubBranch = refName | |
| githubSHA = context.payload.inputs.pull_request_sha | |
| core.info(`workflow_dispatch event: set git info from inputs. inputs: ${JSON.stringify(context.payload.inputs)}`) | |
| } else if (context.eventName === "pull_request" || context.eventName === "pull_request_target" ) { | |
| // For PRs from forks, tag images with `prXXX` to avoid clashes between branches. | |
| const targetRepo = context.payload.repository.full_name; | |
| const prRepo = context.payload.pull_request.head.repo.full_name | |
| const prRef = context.payload.pull_request.head.ref | |
| refSlug = `pr${context.issue.number}`; | |
| refName = (prRepo === targetRepo) ? prRef : refSlug; | |
| refFull = `refs/pull/${context.issue.number}/head` | |
| githubBranch = refName | |
| githubSHA = context.payload.pull_request.head.sha | |
| core.info(`pull request event: set git info from pull_request.head. pr:${prRepo}:${prRef} target:${targetRepo}:${context.ref}`) | |
| prNumber = context.issue.number | |
| } else { | |
| // Other triggers: workflow_dispatch without pull_request_ref, schedule, push... | |
| // refName is 'main' or tag name, so slugification is not necessary. | |
| refSlug = GITHUB_REF_NAME | |
| refName = GITHUB_REF_NAME | |
| refFull = GITHUB_REF | |
| githubTag = GITHUB_REF_TYPE == "tag" ? refName : "" | |
| githubBranch = GITHUB_REF_TYPE == "branch" ? refName : "" | |
| githubSHA = context.sha | |
| core.info(`${context.eventName} event: set git info from context: ${JSON.stringify({GITHUB_REF_NAME, GITHUB_REF_TYPE, sha: context.sha })}`) | |
| } | |
| core.setCommandEcho(true) | |
| core.setOutput('ci_commit_ref_slug', refSlug) | |
| core.setOutput('ci_commit_ref_name', refName) | |
| core.setOutput(`ci_commit_tag`, githubTag) | |
| core.setOutput(`ci_commit_branch`, githubBranch) | |
| core.setOutput(`ref_full`, refFull) | |
| core.setOutput('github_sha', githubSHA) | |
| core.setOutput('pr_number', prNumber) | |
| core.setCommandEcho(false) | |
| # </template: git_info_job> | |
| # <template: e2e_run_job_template> | |
| run_aws_containerd_1_27: | |
| name: "AWS, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "aws;WithoutNAT;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: AWS | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: AWS/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: AWS | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_AWS_ACCESS_KEY: ${{ secrets.LAYOUT_AWS_ACCESS_KEY }} | |
| LAYOUT_AWS_SECRET_ACCESS_KEY: ${{ secrets.LAYOUT_AWS_SECRET_ACCESS_KEY }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_AWS_ACCESS_KEY=${LAYOUT_AWS_ACCESS_KEY:-not_provided} \ | |
| -e LAYOUT_AWS_SECRET_ACCESS_KEY=${LAYOUT_AWS_SECRET_ACCESS_KEY:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: AWS | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_AWS_ACCESS_KEY: ${{ secrets.LAYOUT_AWS_ACCESS_KEY }} | |
| LAYOUT_AWS_SECRET_ACCESS_KEY: ${{ secrets.LAYOUT_AWS_SECRET_ACCESS_KEY }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_AWS_ACCESS_KEY=${LAYOUT_AWS_ACCESS_KEY:-not_provided} \ | |
| -e LAYOUT_AWS_SECRET_ACCESS_KEY=${LAYOUT_AWS_SECRET_ACCESS_KEY:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_aws_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_aws_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "WithoutNAT", | |
| "provider": "AWS", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_azure_containerd_1_27: | |
| name: "Azure, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "azure;Standard;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: Azure | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: Azure/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: Azure | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_AZURE_SUBSCRIPTION_ID: ${{ secrets.LAYOUT_AZURE_SUBSCRIPTION_ID }} | |
| LAYOUT_AZURE_CLIENT_ID: ${{ secrets.LAYOUT_AZURE_CLIENT_ID }} | |
| LAYOUT_AZURE_CLIENT_SECRET: ${{ secrets.LAYOUT_AZURE_CLIENT_SECRET }} | |
| LAYOUT_AZURE_TENANT_ID: ${{ secrets.LAYOUT_AZURE_TENANT_ID }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_AZURE_SUBSCRIPTION_ID=${LAYOUT_AZURE_SUBSCRIPTION_ID:-not_provided} \ | |
| -e LAYOUT_AZURE_CLIENT_ID=${LAYOUT_AZURE_CLIENT_ID:-not_provided} \ | |
| -e LAYOUT_AZURE_CLIENT_SECRET=${LAYOUT_AZURE_CLIENT_SECRET:-not_provided} \ | |
| -e LAYOUT_AZURE_TENANT_ID=${LAYOUT_AZURE_TENANT_ID:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: Azure | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_AZURE_SUBSCRIPTION_ID: ${{ secrets.LAYOUT_AZURE_SUBSCRIPTION_ID }} | |
| LAYOUT_AZURE_CLIENT_ID: ${{ secrets.LAYOUT_AZURE_CLIENT_ID }} | |
| LAYOUT_AZURE_CLIENT_SECRET: ${{ secrets.LAYOUT_AZURE_CLIENT_SECRET }} | |
| LAYOUT_AZURE_TENANT_ID: ${{ secrets.LAYOUT_AZURE_TENANT_ID }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_AZURE_SUBSCRIPTION_ID=${LAYOUT_AZURE_SUBSCRIPTION_ID:-not_provided} \ | |
| -e LAYOUT_AZURE_CLIENT_ID=${LAYOUT_AZURE_CLIENT_ID:-not_provided} \ | |
| -e LAYOUT_AZURE_CLIENT_SECRET=${LAYOUT_AZURE_CLIENT_SECRET:-not_provided} \ | |
| -e LAYOUT_AZURE_TENANT_ID=${LAYOUT_AZURE_TENANT_ID:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_azure_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_azure_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "Standard", | |
| "provider": "Azure", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_gcp_containerd_1_27: | |
| name: "GCP, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "gcp;WithoutNAT;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: GCP | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: GCP/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: GCP | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON: ${{ secrets.LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON=${LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: GCP | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON: ${{ secrets.LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON=${LAYOUT_GCP_SERVICE_ACCOUT_KEY_JSON:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_gcp_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_gcp_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "WithoutNAT", | |
| "provider": "GCP", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_yandex_cloud_containerd_1_27: | |
| name: "Yandex.Cloud, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "yandex-cloud;WithoutNAT;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: Yandex.Cloud | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: Yandex.Cloud/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: Yandex.Cloud | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_YANDEX_CLOUD_ID: ${{ secrets.LAYOUT_YANDEX_CLOUD_ID }} | |
| LAYOUT_YANDEX_FOLDER_ID: ${{ secrets.LAYOUT_YANDEX_FOLDER_ID }} | |
| LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON: ${{ secrets.LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_YANDEX_CLOUD_ID=${LAYOUT_YANDEX_CLOUD_ID:-not_provided} \ | |
| -e LAYOUT_YANDEX_FOLDER_ID=${LAYOUT_YANDEX_FOLDER_ID:-not_provided} \ | |
| -e LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON=${LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: Yandex.Cloud | |
| CRI: Containerd | |
| LAYOUT: WithoutNAT | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_YANDEX_CLOUD_ID: ${{ secrets.LAYOUT_YANDEX_CLOUD_ID }} | |
| LAYOUT_YANDEX_FOLDER_ID: ${{ secrets.LAYOUT_YANDEX_FOLDER_ID }} | |
| LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON: ${{ secrets.LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_YANDEX_CLOUD_ID=${LAYOUT_YANDEX_CLOUD_ID:-not_provided} \ | |
| -e LAYOUT_YANDEX_FOLDER_ID=${LAYOUT_YANDEX_FOLDER_ID:-not_provided} \ | |
| -e LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON=${LAYOUT_YANDEX_SERVICE_ACCOUNT_KEY_JSON:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_yandex-cloud_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_yandex-cloud_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "WithoutNAT", | |
| "provider": "Yandex.Cloud", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_openstack_containerd_1_27: | |
| name: "OpenStack, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "openstack;Standard;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: OpenStack | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: OpenStack/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: OpenStack | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_OS_PASSWORD: ${{ secrets.LAYOUT_OS_PASSWORD }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: OpenStack | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_OS_PASSWORD: ${{ secrets.LAYOUT_OS_PASSWORD }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_openstack_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_openstack_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "Standard", | |
| "provider": "OpenStack", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_vsphere_containerd_1_27: | |
| name: "vSphere, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "vsphere;Standard;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: vSphere | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-vsphere] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: vSphere/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: vSphere | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_VSPHERE_PASSWORD: ${{ secrets.LAYOUT_VSPHERE_PASSWORD }} | |
| LAYOUT_VSPHERE_BASE_DOMAIN: ${{ secrets.LAYOUT_VSPHERE_BASE_DOMAIN }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_VSPHERE_PASSWORD=${LAYOUT_VSPHERE_PASSWORD:-not_provided} \ | |
| -e LAYOUT_VSPHERE_BASE_DOMAIN=${LAYOUT_VSPHERE_BASE_DOMAIN:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: vSphere | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_VSPHERE_PASSWORD: ${{ secrets.LAYOUT_VSPHERE_PASSWORD }} | |
| LAYOUT_VSPHERE_BASE_DOMAIN: ${{ secrets.LAYOUT_VSPHERE_BASE_DOMAIN }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_VSPHERE_PASSWORD=${LAYOUT_VSPHERE_PASSWORD:-not_provided} \ | |
| -e LAYOUT_VSPHERE_BASE_DOMAIN=${LAYOUT_VSPHERE_BASE_DOMAIN:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_vsphere_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_vsphere_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "Standard", | |
| "provider": "vSphere", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_vclouddirector_containerd_1_27: | |
| name: "vCloudDirector, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "vclouddirector;Standard;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: vCloudDirector | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: vCloudDirector/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: vCloudDirector | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_VCD_PASSWORD: ${{ secrets.LAYOUT_VCD_PASSWORD }} | |
| LAYOUT_VCD_USERNAME: ${{ secrets.LAYOUT_VCD_USERNAME }} | |
| LAYOUT_STATIC_BASTION_IP: 80.249.129.56 | |
| LAYOUT_VCD_SERVER: ${{ secrets.LAYOUT_VCD_SERVER }} | |
| LAYOUT_VCD_ORG: ${{ secrets.LAYOUT_VCD_ORG }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_VCD_PASSWORD=${LAYOUT_VCD_PASSWORD:-not_provided} \ | |
| -e LAYOUT_VCD_USERNAME=${LAYOUT_VCD_USERNAME:-not_provided} \ | |
| -e LAYOUT_VCD_SERVER=${LAYOUT_VCD_SERVER:-not_provided} \ | |
| -e LAYOUT_VCD_ORG=${LAYOUT_VCD_ORG:-not_provided} \ | |
| -e LAYOUT_STATIC_BASTION_IP=80.249.129.56 \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: vCloudDirector | |
| CRI: Containerd | |
| LAYOUT: Standard | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_VCD_PASSWORD: ${{ secrets.LAYOUT_VCD_PASSWORD }} | |
| LAYOUT_VCD_USERNAME: ${{ secrets.LAYOUT_VCD_USERNAME }} | |
| LAYOUT_STATIC_BASTION_IP: 80.249.129.56 | |
| LAYOUT_VCD_SERVER: ${{ secrets.LAYOUT_VCD_SERVER }} | |
| LAYOUT_VCD_ORG: ${{ secrets.LAYOUT_VCD_ORG }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_VCD_PASSWORD=${LAYOUT_VCD_PASSWORD:-not_provided} \ | |
| -e LAYOUT_VCD_USERNAME=${LAYOUT_VCD_USERNAME:-not_provided} \ | |
| -e LAYOUT_VCD_SERVER=${LAYOUT_VCD_SERVER:-not_provided} \ | |
| -e LAYOUT_VCD_ORG=${LAYOUT_VCD_ORG:-not_provided} \ | |
| -e LAYOUT_STATIC_BASTION_IP=80.249.129.56 \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_vclouddirector_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_vclouddirector_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "Standard", | |
| "provider": "vCloudDirector", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| # <template: e2e_run_job_template> | |
| run_static_containerd_1_27: | |
| name: "Static, Containerd, Kubernetes 1.27" | |
| needs: | |
| - git_info | |
| outputs: | |
| ssh_master_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_master_connection_string }} | |
| ssh_bastion_connection_string: ${{ steps.check_stay_failed_cluster.outputs.ssh_bastion_connection_string }} | |
| run_id: ${{ github.run_id }} | |
| # need for find state in artifact | |
| cluster_prefix: ${{ steps.setup.outputs.dhctl-prefix }} | |
| ran_for: "static;Static;containerd;1.27" | |
| failed_cluster_stayed: ${{ steps.check_stay_failed_cluster.outputs.failed_cluster_stayed }} | |
| issue_number: ${{ inputs.issue_number }} | |
| install_image_path: ${{ steps.setup.outputs.install-image-path }} | |
| env: | |
| PROVIDER: Static | |
| CRI: Containerd | |
| LAYOUT: Static | |
| KUBERNETES_VERSION: "1.27" | |
| EVENT_LABEL: ${{ github.event.label.name }} | |
| runs-on: [self-hosted, e2e-common] | |
| steps: | |
| # <template: started_at_output> | |
| - name: Job started timestamp | |
| id: started_at | |
| run: | | |
| unixTimestamp=$(date +%s) | |
| echo "started_at=${unixTimestamp}" >> $GITHUB_OUTPUT | |
| # </template: started_at_output> | |
| # <template: checkout_from_event_ref_step> | |
| - name: Checkout sources | |
| uses: actions/[email protected] | |
| with: | |
| ref: ${{ github.event.inputs.pull_request_ref || github.event.ref }} | |
| fetch-depth: 0 | |
| # </template: checkout_from_event_ref_step> | |
| # <template: login_dev_registry_step> | |
| - name: Check dev registry credentials | |
| id: check_dev_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_DEV_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_DEV_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to dev registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_dev_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_DEV_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_DEV_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_DEV_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_dev_registry_step> | |
| # <template: login_rw_registry_step> | |
| - name: Check rw registry credentials | |
| id: check_rw_registry | |
| env: | |
| HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| run: | | |
| if [[ -n $HOST ]]; then | |
| echo "has_credentials=true" >> $GITHUB_OUTPUT | |
| echo "web_registry_path=${{secrets.DECKHOUSE_REGISTRY_HOST }}/deckhouse/site" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Login to rw registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials == 'true' }} | |
| with: | |
| registry: ${{ secrets.DECKHOUSE_REGISTRY_HOST }} | |
| username: ${{ secrets.DECKHOUSE_REGISTRY_USER }} | |
| password: ${{ secrets.DECKHOUSE_REGISTRY_PASSWORD }} | |
| logout: false | |
| - name: Login to Github Container Registry | |
| uses: docker/[email protected] | |
| if: ${{ steps.check_rw_registry.outputs.has_credentials != 'true' }} | |
| with: | |
| registry: ghcr.io | |
| username: ${{ secrets.GHCR_IO_REGISTRY_USER }} | |
| password: ${{ secrets.GHCR_IO_REGISTRY_PASSWORD }} | |
| logout: false | |
| # </template: login_rw_registry_step> | |
| # <template: werf_install_step> | |
| - name: Install werf CLI | |
| uses: werf/actions/install@43075e4ab81952b181d33e125ef15b9c060a782e | |
| with: | |
| channel: ${{env.WERF_CHANNEL}} | |
| # </template: werf_install_step> | |
| - name: Setup | |
| id: setup | |
| env: | |
| DECKHOUSE_REGISTRY_HOST: ${{secrets.DECKHOUSE_REGISTRY_HOST}} | |
| CI_COMMIT_TAG: ${{needs.git_info.outputs.ci_commit_tag}} | |
| CI_COMMIT_BRANCH: ${{needs.git_info.outputs.ci_commit_branch}} | |
| CI_COMMIT_REF_SLUG: ${{needs.git_info.outputs.ci_commit_ref_slug}} | |
| REF_FULL: ${{needs.git_info.outputs.ref_full}} | |
| INITIAL_REF_SLUG: ${{ github.event.inputs.initial_ref_slug }} | |
| MANUAL_RUN: "false" | |
| run: | | |
| # Calculate unique prefix for e2e test. | |
| # GITHUB_RUN_ID is a unique number for each workflow run. | |
| # GITHUB_RUN_ATTEMPT is a unique number for each attempt of a particular workflow run in a repository. | |
| # Add CRI and KUBERNETES_VERSION to create unique directory for each job. | |
| # CRI and PROVIDER values are trimmed to reduce prefix length. | |
| if [[ "${KUBERNETES_VERSION}" == "Automatic" ]] ; then | |
| KUBERNETES_VERSION_SUF="auto" | |
| else | |
| KUBERNETES_VERSION_SUF=${KUBERNETES_VERSION} | |
| fi | |
| DHCTL_PREFIX=$(echo "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-$(echo ${CRI} | head -c 3)-${KUBERNETES_VERSION_SUF}") | |
| if [[ "${MANUAL_RUN}" == "false" ]] ; then | |
| # for jobs which run multiple providers concurrency (daily e2e, for example) | |
| # add provider suffix to prevent "directory already exists" error | |
| DHCTL_PREFIX="${DHCTL_PREFIX}-$(echo ${PROVIDER} | head -c 2)" | |
| fi | |
| # converts to DNS-like (all letters in lower case and replace all dots to dash) | |
| # because it prefix will use for k8s resources names (nodes, for example) | |
| DHCTL_PREFIX=$(echo "$DHCTL_PREFIX" | tr '.' '-' | tr '[:upper:]' '[:lower:]') | |
| # Create tmppath for test script. | |
| TMP_DIR_PATH=/mnt/cloud-layouts/layouts/${DHCTL_PREFIX} | |
| if [[ -d "${TMP_DIR_PATH}" ]] ; then | |
| echo "Temporary dir already exists: ${TMP_DIR_PATH}. ERROR!" | |
| ls -la ${TMP_DIR_PATH} | |
| exit 1 | |
| else | |
| echo "Create temporary dir for job: ${TMP_DIR_PATH}." | |
| mkdir -p "${TMP_DIR_PATH}" | |
| fi | |
| ## Source: ci_templates/build.yml | |
| # Extract REPO_SUFFIX from repository name: trim prefix 'deckhouse/deckhouse-'. | |
| REPO_SUFFIX=${GITHUB_REPOSITORY#deckhouse/deckhouse-} | |
| if [[ $REPO_SUFFIX == $GITHUB_REPOSITORY ]] ; then | |
| # REPO_SUFFIX should be empty for main repo 'deckhouse/deckhouse'. | |
| REPO_SUFFIX= | |
| fi | |
| # Use dev-registry for Git branches. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| # Use rw-registry for Git tags. | |
| SEMVER_REGISTRY_PATH="${DECKHOUSE_REGISTRY_HOST}/deckhouse" | |
| if [[ -z ${DECKHOUSE_REGISTRY_HOST:-} ]] ; then | |
| # DECKHOUSE_REGISTRY_HOST is empty, so this repo is not the main repo. | |
| # Use dev-regisry for branches and Github Container Registry for semver tags. | |
| BRANCH_REGISTRY_PATH="${DEV_REGISTRY_PATH}" | |
| SEMVER_REGISTRY_PATH="${GHA_TEST_REGISTRY_PATH}" | |
| fi | |
| # Prepare initial image tag for deploy/deckhouse to test switching from previous release. | |
| INITIAL_IMAGE_TAG= | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INITIAL_IMAGE_TAG=${INITIAL_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| fi | |
| # Prepare image tag for deploy/deckhouse (DECKHOUSE_IMAGE_TAG option in testing/cloud_layouts/script.sh). | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| # Use it as image tag. Add suffix to not overlap with PRs in main repo. | |
| IMAGE_TAG=${CI_COMMIT_REF_SLUG}${REPO_SUFFIX:+-${REPO_SUFFIX}} | |
| INSTALL_IMAGE_NAME= | |
| if [[ -n ${CI_COMMIT_BRANCH} ]]; then | |
| # CI_COMMIT_REF_SLUG is a 'prNUM' for dev branches or 'main' for default branch. | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${IMAGE_TAG} | |
| fi | |
| if [[ -n ${CI_COMMIT_TAG} ]] ; then | |
| REGISTRY_SUFFIX=$(echo ${WERF_ENV} | tr '[:upper:]' '[:lower:]') # CE/EE/FE -> ce/ee/fe | |
| INSTALL_IMAGE_NAME=${SEMVER_REGISTRY_PATH}/${REGISTRY_SUFFIX}/install:${CI_COMMIT_REF_SLUG} | |
| fi | |
| if [[ -n ${INITIAL_REF_SLUG} ]] ; then | |
| INSTALL_IMAGE_NAME=${BRANCH_REGISTRY_PATH}/install:${INITIAL_IMAGE_TAG} | |
| git fetch origin ${INITIAL_REF_SLUG} | |
| git checkout origin/${INITIAL_REF_SLUG} -- testing/cloud_layouts | |
| fi | |
| SAFE_IMAGE_NAME=$(echo ${INSTALL_IMAGE_NAME} | tr '[:lower:]' '[:upper:]') | |
| echo "Deckhouse Deployment will use install image ${SAFE_IMAGE_NAME} to test Git ref ${REF_FULL}" | |
| # Print image name in uppercase to prevent hiding non-secret registry host stored in secret. | |
| echo "⚓️ [$(date -u)] Pull 'dev/install' image '${SAFE_IMAGE_NAME}'." | |
| docker pull "${INSTALL_IMAGE_NAME}" | |
| IMAGE_INSTALL_PATH="/${INSTALL_IMAGE_NAME#*/}" | |
| echo '::echo::on' | |
| echo "tmp-dir-path=${TMP_DIR_PATH}" >> $GITHUB_OUTPUT | |
| echo "dhctl-log-file=${TMP_DIR_PATH}/dhctl.log" >> $GITHUB_OUTPUT | |
| echo "dhctl-prefix=${DHCTL_PREFIX}" >> $GITHUB_OUTPUT | |
| echo "install-image-name=${INSTALL_IMAGE_NAME}" >> $GITHUB_OUTPUT | |
| echo "deckhouse-image-tag=${IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "initial-image-tag=${INITIAL_IMAGE_TAG}" >> $GITHUB_OUTPUT | |
| echo "install-image-path=${IMAGE_INSTALL_PATH}" >> $GITHUB_OUTPUT | |
| echo '::echo::off' | |
| - name: "Run e2e test: Static/Containerd/1.27" | |
| id: e2e_test_run | |
| timeout-minutes: 80 | |
| env: | |
| PROVIDER: Static | |
| CRI: Containerd | |
| LAYOUT: Static | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| INITIAL_IMAGE_TAG: ${{ steps.setup.outputs.initial-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_OS_PASSWORD: ${{ secrets.LAYOUT_OS_PASSWORD }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh run-test' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh run-test | |
| # </template: e2e_run_template> | |
| - name: Cleanup bootstrapped cluster | |
| if: always() | |
| id: cleanup_cluster | |
| timeout-minutes: 60 | |
| env: | |
| PROVIDER: Static | |
| CRI: Containerd | |
| LAYOUT: Static | |
| KUBERNETES_VERSION: "1.27" | |
| LAYOUT_DECKHOUSE_DOCKERCFG: ${{ secrets.LAYOUT_DECKHOUSE_DOCKERCFG }} | |
| LAYOUT_SSH_KEY: ${{ secrets.LAYOUT_SSH_KEY}} | |
| TMP_DIR_PATH: ${{ steps.setup.outputs.tmp-dir-path}} | |
| PREFIX: ${{ steps.setup.outputs.dhctl-prefix}} | |
| INSTALL_IMAGE_NAME: ${{ steps.setup.outputs.install-image-name }} | |
| DECKHOUSE_IMAGE_TAG: ${{ steps.setup.outputs.deckhouse-image-tag }} | |
| # <template: e2e_run_template> | |
| LAYOUT_OS_PASSWORD: ${{ secrets.LAYOUT_OS_PASSWORD }} | |
| COMMENT_ID: ${{ inputs.comment_id }} | |
| GITHUB_API_SERVER: ${{ github.api_url }} | |
| REPOSITORY: ${{ github.repository }} | |
| DHCTL_LOG_FILE: ${{ steps.setup.outputs.dhctl-log-file}} | |
| GITHUB_TOKEN: ${{secrets.BOATSWAIN_GITHUB_TOKEN}} | |
| run: | | |
| echo "Execute 'script.sh cleanup' via 'docker run', using environment: | |
| INSTALL_IMAGE_NAME=${INSTALL_IMAGE_NAME} | |
| DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} | |
| INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} | |
| PREFIX=${PREFIX} | |
| PROVIDER=${PROVIDER} | |
| CRI=${CRI} | |
| LAYOUT=${LAYOUT} | |
| KUBERNETES_VERSION=${KUBERNETES_VERSION} | |
| TMP_DIR_PATH=${TMP_DIR_PATH} | |
| " | |
| ls -lh $(pwd)/testing | |
| dhctl_log_file="${DHCTL_LOG_FILE}-${PROVIDER}-${LAYOUT}-${CRI}-${KUBERNETES_VERSION}" | |
| echo "DHCTL log file: $dhctl_log_file" | |
| user_runner_id=$(id -u):$(id -g) | |
| echo "user_runner_id $user_runner_id" | |
| docker run --rm \ | |
| -e DECKHOUSE_DOCKERCFG=${LAYOUT_DECKHOUSE_DOCKERCFG} \ | |
| -e PREFIX=${PREFIX} \ | |
| -e DECKHOUSE_IMAGE_TAG=${DECKHOUSE_IMAGE_TAG} \ | |
| -e INITIAL_IMAGE_TAG=${INITIAL_IMAGE_TAG} \ | |
| -e KUBERNETES_VERSION=${KUBERNETES_VERSION} \ | |
| -e CRI=${CRI} \ | |
| -e PROVIDER=${PROVIDER:-not_provided} \ | |
| -e MASTER_CONNECTION_STRING=${SSH_MASTER_CONNECTION_STRING:-} \ | |
| -e LAYOUT=${LAYOUT:-not_provided} \ | |
| -e DHCTL_LOG_FILE="/tmp/$(basename ${dhctl_log_file})" \ | |
| -e SSH_KEY=${LAYOUT_SSH_KEY:-not_provided} \ | |
| -e LAYOUT_OS_PASSWORD=${LAYOUT_OS_PASSWORD:-not_provided} \ | |
| -e USER_RUNNER_ID=${user_runner_id} \ | |
| -v $(pwd)/testing:/deckhouse/testing \ | |
| -v $(pwd)/release.yaml:/deckhouse/release.yaml \ | |
| -v ${TMP_DIR_PATH}:/tmp \ | |
| -w /deckhouse \ | |
| ${INSTALL_IMAGE_NAME} \ | |
| bash /deckhouse/testing/cloud_layouts/script.sh cleanup | |
| # </template: e2e_run_template> | |
| - name: Save dhctl state | |
| id: save_failed_cluster_state | |
| if: ${{ failure() }} | |
| uses: actions/[email protected] | |
| with: | |
| name: failed_cluster_state_static_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.tmp-dir-path}}/dhctl | |
| ${{ steps.setup.outputs.tmp-dir-path}}/*.tfstate | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| - name: Save test results | |
| if: ${{ steps.setup.outputs.dhctl-log-file }} | |
| uses: actions/[email protected] | |
| with: | |
| name: test_output_static_containerd_1_27 | |
| path: | | |
| ${{ steps.setup.outputs.dhctl-log-file}}* | |
| ${{ steps.setup.outputs.tmp-dir-path}}/logs | |
| testing/cloud_layouts/ | |
| !testing/cloud_layouts/**/sshkey | |
| - name: Cleanup temp directory | |
| if: always() | |
| env: | |
| TMPPATH: ${{ steps.setup.outputs.tmppath}} | |
| run: | | |
| echo "Remove temporary directory '${TMPPATH}' ..." | |
| if [[ -d "${TMPPATH}" && ${#TMPPATH} > 1 ]] ; then | |
| rm -rf "${TMPPATH}" | |
| else | |
| echo Not a directory. | |
| fi | |
| if [ -n $USER_RUNNER_ID ]; then | |
| echo "Fix temp directories owner..." | |
| chown -R $USER_RUNNER_ID "$(pwd)/testing" || true | |
| chown -R $USER_RUNNER_ID "/deckhouse/testing" || true | |
| chown -R $USER_RUNNER_ID /tmp || true | |
| else | |
| echo "Fix temp directories permissions..." | |
| chmod -f -R 777 "$(pwd)/testing" || true | |
| chmod -f -R 777 "/deckhouse/testing" || true | |
| chmod -f -R 777 /tmp || true | |
| fi | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' && (github.ref == 'refs/heads/main' && (cancelled() || failure())) }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "cri": "Containerd", | |
| "kube_version": "1.27", | |
| "layout": "Static", | |
| "provider": "Static", | |
| "trigger": "CloudLayoutTestFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Github workflow log for more information", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Cloud Layout Test failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> | |
| # </template: e2e_run_job_template> | |
| send_alert_about_workflow_problem: | |
| name: Send alert about workflow problem | |
| runs-on: ubuntu-latest | |
| needs: ["skip_tests_repos","git_info","run_aws_containerd_1_27","run_azure_containerd_1_27","run_gcp_containerd_1_27","run_yandex_cloud_containerd_1_27","run_openstack_containerd_1_27","run_vsphere_containerd_1_27","run_vclouddirector_containerd_1_27","run_static_containerd_1_27"] | |
| if: ${{ failure() }} | |
| steps: | |
| # <template: send_alert_template> | |
| - name: Check alerting credentials | |
| id: check_alerting | |
| if: always() | |
| env: | |
| KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| if [[ -n $KEY ]]; then echo "has_credentials=true" >> $GITHUB_OUTPUT; fi | |
| - name: Send alert on fail | |
| if: ${{ steps.check_alerting.outputs.has_credentials == 'true' }} | |
| env: | |
| CLOUD_LAYOUT_TESTS_MADISON_KEY: ${{secrets.CLOUD_LAYOUT_TESTS_MADISON_KEY}} | |
| run: | | |
| WORKFLOW_URL="${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}" | |
| echo $WORKFLOW_URL | |
| alertData=$(cat <<EOF | |
| { | |
| "labels": { | |
| "trigger": "DailyE2EWorkflowFailed", | |
| "severity_level": 7 | |
| }, | |
| "annotations": { | |
| "description": "Check Daily e2e workflow log for more information or see another alerts in this group.", | |
| "plk_create_group_if_not_exists/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "plk_grouped_by/cloudlayouttestfailed": "CloudLayoutTestFailedGroup", | |
| "summary": "Daily e2e tests workflow failed", | |
| "plk_link_url/job": "${WORKFLOW_URL}", | |
| "plk_protocol_version": "1", | |
| "plk_link_title_en/job": "Github job run" | |
| } | |
| } | |
| EOF | |
| ) | |
| for (( iter = 1; iter < 60; iter++ )); do | |
| if curl -sS -X "POST" "https://madison.flant.com/api/events/custom/${CLOUD_LAYOUT_TESTS_MADISON_KEY}" -H 'Content-Type: application/json' -d "${alertData}"; then | |
| exit 0 | |
| fi | |
| echo "Alert was not sent. Wait 5 seconds before next attempt" | |
| sleep 5 | |
| done | |
| echo "Alert was not sent. Timeout" | |
| exit 1 | |
| # </template: send_alert_template> |