stackhpc-all-in-one.yml

---
# This reusable workflow deploys a VM on a cloud using Terraform, then deploys
# OpenStack in the VM via Kayobe. Tempest is then used to test the cloud.

name: All in one

on:
  workflow_call:
    inputs:
      kayobe_image:
        description: Kayobe container image
        type: string
        required: true
      os_distribution:
        description: Host OS distribution
        type: string
        default: rocky
      os_release:
        description: Host OS release
        type: string
        default: '9'
      ssh_username:
        description: User for terraform to access the all-in-one VM
        type: string
        default: cloud-user
      neutron_plugin:
        description: Neutron ML2 plugin
        type: string
        required: true
      vm_image_override:
        description: Full name of an image to use instead of the default
        type: string
        default: ""
      vm_interface:
        description: Default network interface name
        type: string
        default: ens3
      vm_flavor:
        description: Flavor for the all-in-one VM
        type: string
        default: en1.medium
      vm_network:
        description: Network for the all-in-one VM
        type: string
        default: stackhpc-ci
      vm_subnet:
        description: Subnet for the all-in-one VM
        type: string
        default: stackhpc-ci
      OS_CLOUD:
        description: Name of cloud in clouds.yaml
        type: string
        required: true
      if:
        description: Whether to run the workflow (workaround for required status checks issue)
        type: boolean
        default: true
      upgrade:
        description: Whether to perform an upgrade
        type: boolean
        default: false
    secrets:
      KAYOBE_VAULT_PASSWORD:
        required: true
      CLOUDS_YAML:
        required: true
      OS_APPLICATION_CREDENTIAL_ID:
        required: true
      OS_APPLICATION_CREDENTIAL_SECRET:
        required: true

jobs:
  # NOTE: Runner needs unzip and nodejs packages.
  all-in-one:
    name: All in one
    if: ${{ inputs.if && !cancelled() }}
    runs-on: arc-skc-aio-runner
    permissions: {}
    env:
      KAYOBE_ENVIRONMENT: ci-aio
      KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
      KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
      # NOTE(upgrade): Reference the PREVIOUS release here.
      PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-2023.1
      # NOTE(upgrade): Reference the PREVIOUS release branch here.
      PREVIOUS_BRANCH: stackhpc/2023.1
    steps:
      - name: Install Package
        uses: ConorMacBride/install-package@main
        with:
          apt: git unzip nodejs

      # If testing upgrade, checkout previous release, otherwise checkout current branch
      - name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }}
          submodules: true

      - name: Output Kayobe image
        id: kayobe_image
        run: |
          if ${{ inputs.upgrade }}; then
            kayobe_image=$PREVIOUS_KAYOBE_IMAGE
          else
            kayobe_image=$KAYOBE_IMAGE
          fi
          echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT

      - name: Make sure dockerd is running and test Docker
        run: |
          docker ps

      - name: Output image tag
        id: image_tag
        run: |
          echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT

      # Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
      - name: Output image name
        id: image_name
        run: |
          if [ -z "${{ inputs.vm_image_override }}" ]; then
            echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
          else
            echo image_name=${{ inputs.vm_image_override }} >> $GITHUB_OUTPUT
          fi

      - name: Install terraform
        uses: hashicorp/setup-terraform@v2

      - name: Initialise terraform
        run: terraform init
        working-directory: ${{ github.workspace }}/terraform/aio

      - name: Generate SSH keypair
        run: ssh-keygen -f id_rsa -N ''
        working-directory: ${{ github.workspace }}/terraform/aio

      - name: Generate clouds.yaml
        run: |
          cat << EOF > clouds.yaml
          ${{ secrets.CLOUDS_YAML }}
          EOF
        working-directory: ${{ github.workspace }}/terraform/aio

      - name: Generate terraform.tfvars
        run: |
          cat << EOF > terraform.tfvars
          ssh_public_key = "id_rsa.pub"
          ssh_username = "${{ env.SSH_USERNAME }}"
          aio_vm_interface = "${{ env.VM_INTERFACE }}"
          aio_vm_name = "${{ env.VM_NAME }}"
          aio_vm_image = "${{ env.VM_IMAGE }}"
          aio_vm_flavor = "${{ env.VM_FLAVOR }}"
          aio_vm_network = "${{ env.VM_NETWORK }}"
          aio_vm_subnet = "${{ env.VM_SUBNET }}"
          aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
          aio_vm_tags = ${{ env.VM_TAGS }}
          EOF
        working-directory: ${{ github.workspace }}/terraform/aio
        env:
          SSH_USERNAME: "${{ inputs.ssh_username }}"
          VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
          VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
          VM_FLAVOR: ${{ inputs.vm_flavor }}
          VM_NETWORK: ${{ inputs.vm_network }}
          VM_SUBNET: ${{ inputs.vm_subnet }}
          VM_INTERFACE: ${{ inputs.vm_interface }}
          VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }}
          VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'

      - name: Terraform Plan
        run: terraform plan
        working-directory: ${{ github.workspace }}/terraform/aio
        env:
          OS_CLOUD: ${{ inputs.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Terraform Apply
        id: tf_apply
        run: |
          for attempt in $(seq 5); do
              if terraform apply -auto-approve; then
                  echo "Created infrastructure on attempt $attempt"
                  exit 0
              fi
              echo "Failed to create infrastructure on attempt $attempt"
              sleep 10
              terraform destroy -auto-approve
              sleep 60
          done
          echo "Failed to create infrastructure after $attempt attempts"
          exit 1
        working-directory: ${{ github.workspace }}/terraform/aio
        env:
          OS_CLOUD: ${{ inputs.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}

      - name: Get Terraform outputs
        id: tf_outputs
        run: |
          terraform output -json
        working-directory: ${{ github.workspace }}/terraform/aio

      - name: Write Terraform outputs
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml
          ${{ steps.tf_outputs.outputs.stdout }}
          EOF

      - name: Write Terraform network config
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml

          admin_oc_net_name: admin
          admin_cidr: "{{ access_cidr.value }}"
          admin_allocation_pool_start: 0.0.0.0
          admin_allocation_pool_end: 0.0.0.0
          admin_gateway: "{{ access_gw.value }}"
          admin_bootproto: dhcp
          admin_ips:
            controller0: "{{ access_ip_v4.value }}"
          EOF

      - name: Write Terraform network interface config
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
          admin_interface: "{{ access_interface.value }}"
          EOF

      - name: Write all-in-one scenario config
        run: |
          cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/zz-aio-scenario.yml
          ---
          os_distribution: ${{ env.OS_DISTRIBUTION }}
          os_release: "${{ env.OS_RELEASE }}"
          kolla_enable_ovn: ${{ env.ENABLE_OVN }}
          EOF
        env:
          ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
          OS_DISTRIBUTION: ${{ inputs.os_distribution }}
          OS_RELEASE: ${{ inputs.os_release }}

      # Use a heredoc to define a multiline string output
      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
      - name: Set SSH key output
        id: ssh_key
        run: |
          echo "ssh_key<<EOF" >> $GITHUB_OUTPUT
          cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      # The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
      - name: Pull previous Kayobe image
        run: |
          docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
        if: inputs.upgrade

      # The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
      - name: Pull current Kayobe image
        run: |
          docker image pull $KAYOBE_IMAGE

      - name: Reboot
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml -e reboot_with_bootstrap_user=true
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Run growroot
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Host configure
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Service deploy
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Configure aio resources
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Change rabbit queues from HA to Quorum
        run: sed -i -e 's/om_enable_rabbitmq_high_availability: true/om_enable_rabbitmq_high_availability: false' -e 's/om_enable_rabbitmq_quorum_queues: false/om_enable_rabbitmq_quorum_queues: true' etc/kayobe/environments/ci-aio/kolla/globals.yml
        if: inputs.upgrade

      - name: Migrate RabbitMQ queues
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml --tags rabbit-queue-migration
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: inputs.upgrade

      # If testing upgrade, checkout the current release branch
      # Stash changes to tracked files, and set clean=false to avoid removing untracked files.
      - name: Stash config changes
        run: git stash
        if: inputs.upgrade

      - name: Checkout current release config
        uses: actions/checkout@v4
        with:
          submodules: true
          clean: false
        if: inputs.upgrade

      - name: Pop stashed config changes
        run: git stash pop
        if: inputs.upgrade

      # Now begin upgrade
      - name: Host upgrade
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: inputs.upgrade

      - name: Host configure
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: inputs.upgrade

      - name: Service upgrade
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: inputs.upgrade

      - name: Ensure we have IP on breth1 to reach the instances
        # NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances
        # we can lose the IP address that allows us to connect to the instances. This playbook
        # also fixes that issue.
        run: |
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            ${{ steps.kayobe_image.outputs.kayobe_image }} \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: inputs.upgrade

      - name: Tempest tests
        id: tempest
        run: |
          mkdir -p tempest-artifacts
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: StackHPC OpenStack tests
        id: stackhpc-openstack-tests
        continue-on-error: true
        run: |
          mkdir -p sot-results
          docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -v $(pwd)/sot-results:/stack/sot-results \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml'
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}

      - name: Collect diagnostic information
        id: diagnostics
        run: |
          mkdir -p diagnostics
          sudo -E docker run -t --rm \
            -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
            -v $(pwd)/diagnostics:/stack/diagnostics \
            -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
            $KAYOBE_IMAGE \
            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml'
        env:
          KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
        if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }}

      - name: Upload test result artifacts
        uses: actions/upload-artifact@v4
        with:
          name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
          path: |
            diagnostics/
            tempest-artifacts/
            sot-results/
        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}

      - name: Fail if any Tempest tests failed
        run: |
          test $(wc -l < tempest-artifacts/failed-tests) -lt 1

      - name: Fail if any StackHPC OpenStack tests failed
        run: |
          echo "Some StackHPC OpenStack tests failed."
          echo "See HTML results artifact (sot-results) for details."
          exit 1
        if: steps.stackhpc-openstack-tests.outcome == 'failure'

      - name: Destroy
        run: terraform destroy -auto-approve
        working-directory: ${{ github.workspace }}/terraform/aio
        env:
          OS_CLOUD: ${{ inputs.OS_CLOUD }}
          OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
          OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
        if: always()