diff --git a/.github/workflows/flowzone.yml b/.github/workflows/flowzone.yml index 94c333f21..f0e63564b 100644 --- a/.github/workflows/flowzone.yml +++ b/.github/workflows/flowzone.yml @@ -50,6 +50,10 @@ jobs: deploy-ami: true deploy-esr: false finalize-hostapp: "no" + run-tests: true + test-environment: balenaos-balenamachine + test-workers: qemu + # depends on Flowzone having already published the helper images # yocto-build: diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 9184ee644..643696ad2 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -3,6 +3,14 @@ name: "Yocto" on: workflow_call: secrets: + # API key names should be descriptive telling us what company, what for, where from, what's the key. + # Let's say, XCompany wants to have test, then keys could be: XCOMPANY_AUTOKIT_BM_ACCESS_TOKEN + AUTOKIT_BALENACLOUD_API_KEY: + description: balena API key for Leviathan testing from BalenaCloud + required: false + AUTOKIT_BALENAMACHINE_API_KEY: + description: balena API key for Leviathan testing from balena-os BM instance + required: false BALENA_API_KEY: description: balena API key for deploying releases # TODO: Different for staging and prod - add another Secret for staging key required: false @@ -59,6 +67,11 @@ on: required: false type: string default: staging + test-environment: + description: The BalenaCloud environment you want tests to target. Available options - production, staging, balenaos-balenamachine + required: false + type: string + default: production deploy-s3: description: Whether to deploy images to S3 required: false @@ -112,6 +125,26 @@ on: required: false type: string default: "us-east-1" + test-suites: + description: comma-separated list of test suites to run + required: false + type: string + default: "os","hup","cloud" + test-workers: + description: comma-separated list of worker types to use for testing. Example - qemu, testbot + required: false + type: string + default: 'qemu','testbot' + BALENACLOUD_APP_NAME: + description: "Testbot fleet for finding available Leviathan workers. Not used for QEMU workers. Can accept a list of apps separated by commas." + type: string + required: false + default: "balena/testbot-rig, balena/testbot-rig-partners" + BALENACLOUD_ORG: + description: "It's weird, please remove it" + type: string + required: false + default: testbot # https://docs.github.com/en/actions/using-jobs/using-concurrency concurrency: @@ -868,4 +901,127 @@ jobs: ## ADD LEVIATHAN JOB HERE, download artifacts "build-artifacts" ## RUN if ${{ inputs.run-tests }} -## Have to re-check out \ No newline at end of file +## Have to re-check out +############################## + # Leviathan Test + ############################## + test: + runs-on: actuated-8cpu-8gb + needs: build + if: ${{ inputs.run-tests }} == true + # runs-on: ${{ fromJSON(inputs.runs-on) }} + environment: ${{ inputs.test-environment }} + + defaults: + run: + working-directory: . + shell: bash --noprofile --norc -eo pipefail -x {0} + + strategy: + fail-fast: false + matrix: + DEVICE_TYPE: ${{ needs.build.outputs.device_slug }} + ENVIRONMENT: ${{ inputs.test-environment }} + WORKER_TYPE: ${{ fromJson(format('[{0}]', inputs.test_workers)) }} + TEST_SUITE: ${{ fromJson(format('[{0}]', inputs.test_suites)) }} + steps: + # https://github.com/actions/checkout + - name: Clone device repository + uses: actions/checkout@v4.1.1 + with: + repository: ${{ inputs.device-repo }} + token: ${{ secrets.GITHUB_TOKEN }} + ref: ${{ inputs.device-repo-ref }} + submodules: true + # fetch-depth: 1 + fetch-tags: true + + - name: Device repository check + run: | + if [ "$(yq '.type' repo.yml)" != "yocto-based OS image" ]; then + echo "::error::Repository does not appear to be of type 'yocto-based OS image'" + exit 1 + fi + - name: Update meta-balena submodule to ${{ inputs.meta-balena-ref }} + if: inputs.meta-balena-ref != '' + working-directory: ./layers/meta-balena + run: | + git config --add remote.origin.fetch '+refs/pull/*:refs/remotes/origin/pr/*' + git fetch --all + git checkout --force "${{ inputs.meta-balena-ref }}" + git submodule update --init --recursive + - name: Setup env variables for Prod + if: inputs.environment == 'production' + run: | + echo "BALENACLOUD_API_KEY=${{ secrets.AUTOKIT_BALENACLOUD_API_KEY }}" >> $GITHUB_ENV + + - name: Setup env variables for balena-os balenamachine + if: inputs.environment == 'balenaos-balenamachine' + run: | + echo "BALENACLOUD_API_KEY=${{ secrets.AUTOKIT_BALENAMACHINE_API_KEY }}" >> $GITHUB_ENV + - name : Setup Leviathan Environement + run: | + # create "workspace" directory + echo LEVIATHAN_WORKSPACE="${WORKSPACE}/leviathan-workspace" >> $GITHUB_ENV + mkdir -p ${LEVIATHAN_WORKSPACE} + # Set Leviathan root location + echo LEVIATHAN_ROOT="${WORKSPACE}/layers/meta-balena/tests/leviathan" >> $GITHUB_ENV + # Set suites location + echo LEVIATHAN_SUITES="${WORKSPACE}/layers/meta-balena/tests/suites/${{ matrix.TEST_SUITE }}" >> $GITHUB_ENV + # Copy config.js to leviathan workspace, and gzip (leviathan expects gzipped image) + cp ${LEVIATHAN_SUITES}/config.js ${LEVIATHAN_WORKSPACE}/config.js + # Create reports folder + echo LEVIATHAN_REPORTS="${WORKSPACE/leviathan-reports" >> $GITHUB_ENV + mkdir -p ${LEVIATHAN_REPORTS} + + # Images need to end up in workspace folder and need to have correct names as well + # If are not able to tweak, HUP suite to use hostapp instead then + # We need to do a per matrix addition to make sure we, + # Pull balena.img.gz for os, cloud + # Pull balena-image.docs for HUP with exactly these names + ## Ryan + - name: Fetch artifacts from build job + uses: actions/download-artifact@v4 + with: + name: build-artifacts + path: ${{ env.LEVIATHAN_WORKSPACE }} + + - name: BalenaOS Leviathan Tests + # Path to action + uses: ./ + env: + BALENACLOUD_API_KEY: ${{ env.BALENACLOUD_API_KEY }} + BALENACLOUD_API_URL: ${{ vars.BALENA_URL }} + BALENACLOUD_APP_NAME: ${{ inputs.BALENACLOUD_APP_NAME }} + BALENACLOUD_ORG: ${{ inputs.BALENACLOUD_ORG }} + BALENACLOUD_SSH_PORT: ${{ vars.BALENACLOUD_SSH_PORT }} + BALENACLOUD_SSH_URL: ${{ vars.BALENACLOUD_SSH_URL }} + DEVICE_TYPE: ${{ matrix.DEVICE_TYPE }} + LEVIATHAN_ROOT: ${{ env.LEVIATHAN_ROOT }} + QEMU_CPUS: 4 + QEMU_MEMORY: "4G" + REPORTS: ${{ env.LEVIATHAN_REPORTS }} + SUITES: ${{ env.LEVIATHAN_SUITES }} + WORKER_TYPE: ${{ matrix.WORKER_TYPE }} + WORKSPACE: ${{ env.LEVIATHAN_WORKSPACE }} + + - name: "[Secureboot] BalenaOS Leviathan Tests" + if: inputs.sign-image == true + uses: ./ + env: + BALENACLOUD_API_KEY: ${{ env.BALENACLOUD_API_KEY }} + BALENACLOUD_API_URL: ${{ vars.BALENA_URL }} + BALENACLOUD_APP_NAME: ${{ inputs.BALENACLOUD_APP_NAME }} + BALENACLOUD_ORG: ${{ inputs.BALENACLOUD_ORG }} + BALENACLOUD_SSH_PORT: ${{ vars.BALENACLOUD_SSH_PORT }} + BALENACLOUD_SSH_URL: ${{ vars.BALENACLOUD_SSH_URL }} + DEVICE_TYPE: ${{ matrix.DEVICE_TYPE }} + LEVIATHAN_ROOT: ${{ env.LEVIATHAN_ROOT }} + QEMU_CPUS: 4 + QEMU_MEMORY: "4G" + REPORTS: ${{ env.LEVIATHAN_REPORTS }} + SUITES: ${{ env.LEVIATHAN_SUITES }} + WORKER_TYPE: ${{ matrix.WORKER_TYPE }} + WORKSPACE: ${{ env.LEVIATHAN_WORKSPACE }} + QEMU_SECUREBOOT: 1 + FLASHER_SECUREBOOT: 1