From 76748d5cfa0891726361c58a9ca92fcd11b3d256 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 17 Sep 2024 14:34:08 +0100 Subject: [PATCH] CI: Use GitHub per-environment variables and secrets to decouple from Leafcloud We are planning to move at least some of our CI workloads back onto SMS lab. Currently various Leafcloud resource names are hard-coded into our workflows, making it difficult to support a different cloud. This change extracts cloud-specific information into GitHub per-environment secrets and variables. A Leafcloud environment has been created in the GitHub settings for this repository, and the variables referenced here have been created within it. --- .../workflows/overcloud-host-image-build.yml | 17 ++++++----- .../workflows/overcloud-host-image-upload.yml | 21 +++++++------- .github/workflows/stackhpc-all-in-one.yml | 29 +++++-------------- .github/workflows/stackhpc-ci-cleanup.yml | 5 ++-- .github/workflows/stackhpc-pull-request.yml | 10 ------- 5 files changed, 30 insertions(+), 52 deletions(-) diff --git a/.github/workflows/overcloud-host-image-build.yml b/.github/workflows/overcloud-host-image-build.yml index 1b6c2e5f9..c473c0e42 100644 --- a/.github/workflows/overcloud-host-image-build.yml +++ b/.github/workflows/overcloud-host-image-build.yml @@ -41,6 +41,7 @@ jobs: overcloud-host-image-build: name: Build overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: Leafcloud runs-on: arc-skc-host-image-builder-runner permissions: {} steps: @@ -121,10 +122,10 @@ jobs: aio_vm_name = "skc-host-image-builder" # Must be a Rocky Linux 9 host to successfully build all images # This MUST NOT be an LVM image. It can cause confusing conficts with the built image. - aio_vm_image = "Rocky-9-GenericCloud-Base-9.3-20231113.0.x86_64.qcow2" - aio_vm_flavor = "en1.medium" - aio_vm_network = "stackhpc-ci" - aio_vm_subnet = "stackhpc-ci" + aio_vm_image = "${{ vars.HOST_IMAGE_BUILD_IMAGE }}" + aio_vm_flavor = "${{ vars.HOST_IMAGE_BUILD_FLAVOR }}" + aio_vm_network = "${{ vars.HOST_IMAGE_BUILD_NETWORK }}" + aio_vm_subnet = "${{ vars.HOST_IMAGE_BUILD_SUBNET }}" aio_vm_interface = "eth0" EOF working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio @@ -133,7 +134,7 @@ jobs: run: terraform plan working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio env: - OS_CLOUD: "openstack" + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} @@ -153,7 +154,7 @@ jobs: exit 1 working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio env: - OS_CLOUD: "openstack" + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} @@ -511,7 +512,7 @@ jobs: run: terraform destroy -auto-approve working-directory: ${{ github.workspace }}/src/kayobe-config/terraform/aio env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} - if: always() \ No newline at end of file + if: always() diff --git a/.github/workflows/overcloud-host-image-upload.yml b/.github/workflows/overcloud-host-image-upload.yml index e95531564..b205f5a5e 100644 --- a/.github/workflows/overcloud-host-image-upload.yml +++ b/.github/workflows/overcloud-host-image-upload.yml @@ -43,6 +43,7 @@ jobs: overcloud-host-image-upload: name: Upload overcloud host images if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: Leafcloud runs-on: arc-skc-host-image-builder-runner permissions: {} steps: @@ -106,7 +107,7 @@ jobs: openstack image show \ overcloud-centos-8-stream-${{ steps.centos_8_stream_image_tag.outputs.image_tag }} env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} continue-on-error: true @@ -134,7 +135,7 @@ jobs: --private \ --progress env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.centos && steps.centos_8_stream_image_exists.outcome == 'failure' @@ -151,7 +152,7 @@ jobs: openstack image show \ overcloud-rocky-8-${{ steps.rocky_8_image_tag.outputs.image_tag }} env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} continue-on-error: true @@ -179,7 +180,7 @@ jobs: --private \ --progress env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky8 && steps.rocky_8_image_exists.outcome == 'failure' @@ -196,7 +197,7 @@ jobs: openstack image show \ overcloud-rocky-9-${{ steps.rocky_9_image_tag.outputs.image_tag }} env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} continue-on-error: true @@ -224,7 +225,7 @@ jobs: --private \ --progress env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.rocky9 && steps.rocky_9_image_exists.outcome == 'failure' @@ -241,7 +242,7 @@ jobs: openstack image show \ overcloud-ubuntu-focal-${{ steps.ubuntu_focal_image_tag.outputs.image_tag }} env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} continue-on-error: true @@ -269,7 +270,7 @@ jobs: --private \ --progress env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-focal && steps.ubuntu_focal_image_exists.outcome == 'failure' @@ -286,7 +287,7 @@ jobs: openstack image show \ overcloud-ubuntu-jammy-${{ steps.ubuntu_jammy_image_tag.outputs.image_tag }} env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} continue-on-error: true @@ -314,7 +315,7 @@ jobs: --private \ --progress env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: inputs.ubuntu-jammy && steps.ubuntu_jammy_image_exists.outcome == 'failure' diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml index 4526e1359..156282ce7 100644 --- a/.github/workflows/stackhpc-all-in-one.yml +++ b/.github/workflows/stackhpc-all-in-one.yml @@ -35,22 +35,6 @@ on: description: Default network interface name type: string default: ens3 - vm_flavor: - description: Flavor for the all-in-one VM - type: string - default: en1.medium - vm_network: - description: Network for the all-in-one VM - type: string - default: stackhpc-ci - vm_subnet: - description: Subnet for the all-in-one VM - type: string - default: stackhpc-ci - OS_CLOUD: - description: Name of cloud in clouds.yaml - type: string - required: true if: description: Whether to run the workflow (workaround for required status checks issue) type: boolean @@ -70,6 +54,7 @@ jobs: all-in-one: name: All in one if: ${{ inputs.if && !cancelled() }} + environment: Leafcloud runs-on: arc-skc-aio-runner permissions: {} env: @@ -152,9 +137,9 @@ jobs: SSH_USERNAME: "${{ inputs.ssh_username }}" VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}" VM_IMAGE: ${{ steps.image_name.outputs.image_name }} - VM_FLAVOR: ${{ inputs.vm_flavor }} - VM_NETWORK: ${{ inputs.vm_network }} - VM_SUBNET: ${{ inputs.vm_subnet }} + VM_FLAVOR: ${{ vars.AIO_FLAVOR }} + VM_NETWORK: ${{ vars.AIO_NETWORK }} + VM_SUBNET: ${{ vars.AIO_SUBNET }} VM_INTERFACE: ${{ inputs.vm_interface }} VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]' @@ -162,7 +147,7 @@ jobs: run: terraform plan working-directory: ${{ github.workspace }}/terraform/aio env: - OS_CLOUD: ${{ inputs.OS_CLOUD }} + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} @@ -183,7 +168,7 @@ jobs: exit 1 working-directory: ${{ github.workspace }}/terraform/aio env: - OS_CLOUD: ${{ inputs.OS_CLOUD }} + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} @@ -345,7 +330,7 @@ jobs: run: terraform destroy -auto-approve working-directory: ${{ github.workspace }}/terraform/aio env: - OS_CLOUD: ${{ inputs.OS_CLOUD }} + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} if: always() diff --git a/.github/workflows/stackhpc-ci-cleanup.yml b/.github/workflows/stackhpc-ci-cleanup.yml index a14b2970c..08eaecead 100644 --- a/.github/workflows/stackhpc-ci-cleanup.yml +++ b/.github/workflows/stackhpc-ci-cleanup.yml @@ -9,6 +9,7 @@ jobs: ci-cleanup: name: Clean up stale CI resources if: github.repository == 'stackhpc/stackhpc-kayobe-config' + environment: Leafcloud runs-on: ubuntu-latest permissions: {} steps: @@ -52,7 +53,7 @@ jobs: done exit $result env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} @@ -72,7 +73,7 @@ jobs: done exit $result env: - OS_CLOUD: openstack + OS_CLOUD: ${{ vars.OS_CLOUD }} OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} diff --git a/.github/workflows/stackhpc-pull-request.yml b/.github/workflows/stackhpc-pull-request.yml index ce9243ba2..b1304bb2e 100644 --- a/.github/workflows/stackhpc-pull-request.yml +++ b/.github/workflows/stackhpc-pull-request.yml @@ -85,7 +85,6 @@ jobs: with: kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} neutron_plugin: ovs - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -99,7 +98,6 @@ jobs: with: kayobe_image: ${{ needs.build-kayobe-image.outputs.kayobe_image }} neutron_plugin: ovn - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -115,7 +113,6 @@ jobs: os_distribution: rocky os_release: "8" neutron_plugin: ovs - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -131,7 +128,6 @@ jobs: os_distribution: rocky os_release: "8" neutron_plugin: ovn - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -148,7 +144,6 @@ jobs: os_release: focal ssh_username: ubuntu neutron_plugin: ovs - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -165,7 +160,6 @@ jobs: os_release: focal ssh_username: ubuntu neutron_plugin: ovn - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -182,7 +176,6 @@ jobs: os_release: jammy ssh_username: ubuntu neutron_plugin: ovs - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -199,7 +192,6 @@ jobs: os_release: jammy ssh_username: ubuntu neutron_plugin: ovn - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -216,7 +208,6 @@ jobs: os_release: "9" ssh_username: cloud-user neutron_plugin: ovs - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }} @@ -233,7 +224,6 @@ jobs: os_release: "9" ssh_username: cloud-user neutron_plugin: ovn - OS_CLOUD: openstack if: ${{ needs.check-changes.outputs.aio == 'true' }} secrets: inherit if: ${{ ! failure() && ! cancelled() && github.repository == 'stackhpc/stackhpc-kayobe-config' }}