build_test_images.yaml

name: Build, publish and test images

on:
  workflow_call:
  pull_request: # temp hack!!

jobs:
  read_builds:
    runs-on: ubuntu-latest
    outputs:
      builds: ${{ steps.builds-as-json.outputs.builds }}
    steps:
      - name: Check out the repository
        uses: actions/checkout@v3

      - name: Install script dependencies
        run: pip install -r ./requirements.txt

      - name: Get builds as JSON
        id: builds-as-json
        run: ./bin/builds-as-json

  build_images:
    needs: [read_builds]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        #include: ${{ fromJson(needs.read_builds.outputs.builds) }}
        include: ${{ fromJson(needs.read_builds.outputs.builds) }}
    name: ${{ matrix.name }}
    permissions:
      contents: read
      packages: write
      id-token: write # needed for signing the images with GitHub OIDC Token
    steps:
      - name: Check out the repository
        uses: actions/checkout@v3
        with:
          submodules: recursive

      - name: Write OpenStack credentials
        run: echo "$OS_CLOUDS" > ./clouds.yaml
        env:
          OS_CLOUDS: ${{ secrets.OS_CLOUDS }}

      - name: Set up Packer environment
        run: ./bin/setup
        env:
          PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Build image
        id: build-image
        run: ./bin/build-image
        env:
          OS_CLOUD: openstack
          PACKER_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ENVIRONMENT: arcus
          PACKER_TEMPLATE: ${{ matrix.template }}
          ENV_VAR_FILES: ${{ matrix.var-files }}

      - name: Install cosign
        uses: sigstore/cosign-installer@v3.4.0

      - name: Publish image
        id: publish-image
        run: ./bin/publish-image
        env:
          OS_CLOUD: openstack
          ENVIRONMENT: arcus
          ENV_VAR_FILES: ${{ matrix.var-files }}
          IMAGE_ID: ${{ steps.build-image.outputs.image-id }}
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}

      - name: Write matrix outputs
        uses: cloudposse/github-action-matrix-outputs-write@0.4.2
        with:
          matrix-step-name: ${{ github.job }}
          matrix-key: ${{ matrix.name }}
          outputs: |-
            name: ${{ steps.publish-image.outputs.image-name }}
            url: ${{ steps.publish-image.outputs.image-url }}
            checksum: ${{ steps.publish-image.outputs.image-checksum }}
            cosign-bundle: ${{ steps.publish-image.outputs.cosign-bundle }}
            manifest-extra: ${{ steps.build-image.outputs.manifest-extra }}

  publish_manifest:
    # this job should always run, but needs to run after the build matrix
    needs: [build_images]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    outputs:
      manifest-url-encoded: ${{ steps.encode-manifest-url.outputs.encoded }}
    steps:
      - name: Check out the repository
        uses: actions/checkout@v3

      - name: Read matrix outputs
        id: matrix-outputs
        uses: cloudposse/github-action-matrix-outputs-read@0.1.1
        with:
          matrix-step-name: build_images

      - name: Write outputs
        uses: DamianReeves/write-file-action@0a7fcbe1960c53fc08fe789fa4850d24885f4d84
        with:
          path: build-outputs.json
          write-mode: overwrite
          contents: ${{ steps.matrix-outputs.outputs.result }}

      - name: Generate manifest
        run: ./bin/generate-manifest
        env:
          BUILD_OUTPUTS_FILE: ./build-outputs.json
          MANIFEST_FILE: ./manifest.json

      - name: Install s3cmd
        run: |
          sudo apt-get update -y
          sudo apt-get install -y s3cmd

      - name: Publish manifest to S3
        id: publish-manifest
        run: ./bin/publish-manifest
        env:
          MANIFEST_FILE: ./manifest.json
          ENVIRONMENT: arcus
          ENV_VAR_FILES: common
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}

      # The manifest URL that publish-manifest outputs is a signed URL
      # This means that it contains the S3 access key which, although it does not necessarily need
      # to be kept secret, is provided using a GitHub secret
      # GitHub does not allow outputs that include secrets to be transferred between jobs
      # To get around this, we encrypt the manifest URL using GPG and use that as the output of this workflow
      - name: Encode manifest URL using GPG
        id: encode-manifest-url
        run: |
          result=$(gpg --symmetric --batch --passphrase "${PASSPHRASE}" --output - <(echo "${INPUT}") | base64 -w0)
          echo "encoded=${result}" >> $GITHUB_OUTPUT
        env:
          PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
          INPUT: ${{ steps.publish-manifest.outputs.manifest-url }}

  run_azimuth_tests:
    needs: [publish_manifest]
    runs-on: ubuntu-latest
    steps:
      # The manifest URL that publish-manifest outputs is a signed URL
      # This means that it contains the S3 access key which, although it does not necessarily need
      # to be kept secret, is provided using a GitHub secret
      # GitHub does not allow outputs that include secrets to be transferred between jobs
      # To get around this, the manifest URL is encrypted using GPG that we must now decrypt to use
      - name: Decode manifest URL using GPG
        id: decode-manifest-url
        run: |
          result=$(gpg --decrypt --quiet --batch --passphrase "${PASSPHRASE}" --output - <(echo "${INPUT}" | base64 -d))
          echo "decoded=${result}" >> $GITHUB_OUTPUT
        env:
          PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
          INPUT: ${{ needs.publish_manifest.outputs.manifest-url-encoded }}

      # Check out the configuration repository
      - name: Set up Azimuth environment
        uses: stackhpc/azimuth-config/.github/actions/setup@devel
        with:
          os-clouds: ${{ secrets.OS_CLOUDS }}
          environment-prefix: images-ci
          # Use the manifest that we just built
          # We want to run all the CaaS tests except Slurm
          # We want to run the Kubernetes tests _for all Kubernetes versions_
          # We don't need to run the apps tests
          extra-vars: |
            community_images_azimuth_images_manifest_url: ${{ steps.decode-manifest-url.outputs.decoded }}
            generate_tests_caas_test_case_slurm_enabled: false
            generate_tests_kubernetes_test_cases_latest_only: false
            generate_tests_kubernetes_apps_suite_enabled: false

      - name: Provision Azimuth
        uses: stackhpc/azimuth-config/.github/actions/provision@devel

      - name: Run Azimuth tests
        uses: stackhpc/azimuth-config/.github/actions/test@devel

      - name: Destroy Azimuth
        uses: stackhpc/azimuth-config/.github/actions/destroy@devel
        if: ${{ always() }}

  # Purge the images that we just tested from OpenStack
  purge_images:
    needs: [run_azimuth_tests]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Install s3cmd
        run: |
          sudo apt-get update -y
          sudo apt-get install -y s3cmd

      - name: Install script dependencies
        run: pip install -r ./requirements.txt

      - name: Write OpenStack credentials
        run: echo "$OS_CLOUDS" > ./clouds.yaml
        env:
          OS_CLOUDS: ${{ secrets.OS_CLOUDS }}

      - name: Purge images for manifest
        run: |
          source ./bin/env-vars
          ./bin/purge-images "${GITHUB_SHA}.manifest"
        env:
          REPO_ROOT: ${{ github.workspace }}
          OS_CLOUD: openstack
          ENVIRONMENT: arcus
          ENV_VAR_FILES: common
          S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }}