Revert "Use Quorum queues by default in RabbitMQ" #7
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
--- | ||
# This reusable workflow deploys a VM on a cloud using Terraform, then deploys | ||
# OpenStack in the VM via Kayobe. Tempest is then used to test the cloud. | ||
name: All in one | ||
on: | ||
workflow_call: | ||
inputs: | ||
kayobe_image: | ||
description: Kayobe container image | ||
type: string | ||
required: true | ||
os_distribution: | ||
description: Host OS distribution | ||
type: string | ||
default: rocky | ||
os_release: | ||
description: Host OS release | ||
type: string | ||
default: '9' | ||
ssh_username: | ||
description: User for terraform to access the all-in-one VM | ||
type: string | ||
default: cloud-user | ||
neutron_plugin: | ||
description: Neutron ML2 plugin | ||
type: string | ||
required: true | ||
vm_image_override: | ||
description: Full name of an image to use instead of the default | ||
type: string | ||
default: "" | ||
vm_interface: | ||
description: Default network interface name | ||
type: string | ||
default: ens3 | ||
vm_flavor: | ||
description: Flavor for the all-in-one VM | ||
type: string | ||
default: en1.medium | ||
vm_network: | ||
description: Network for the all-in-one VM | ||
type: string | ||
default: stackhpc-ci | ||
vm_subnet: | ||
description: Subnet for the all-in-one VM | ||
type: string | ||
default: stackhpc-ci | ||
OS_CLOUD: | ||
description: Name of cloud in clouds.yaml | ||
type: string | ||
required: true | ||
if: | ||
description: Whether to run the workflow (workaround for required status checks issue) | ||
type: boolean | ||
default: true | ||
upgrade: | ||
description: Whether to perform an upgrade | ||
type: boolean | ||
default: false | ||
secrets: | ||
KAYOBE_VAULT_PASSWORD: | ||
required: true | ||
CLOUDS_YAML: | ||
required: true | ||
OS_APPLICATION_CREDENTIAL_ID: | ||
required: true | ||
OS_APPLICATION_CREDENTIAL_SECRET: | ||
required: true | ||
jobs: | ||
# NOTE: Runner needs unzip and nodejs packages. | ||
all-in-one: | ||
name: All in one | ||
if: ${{ inputs.if && !cancelled() }} | ||
runs-on: arc-skc-aio-runner | ||
permissions: {} | ||
env: | ||
KAYOBE_ENVIRONMENT: ci-aio | ||
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }} | ||
KAYOBE_IMAGE: ${{ inputs.kayobe_image }} | ||
# NOTE(upgrade): Reference the PREVIOUS release here. | ||
PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-2023.1 | ||
# NOTE(upgrade): Reference the PREVIOUS release branch here. | ||
PREVIOUS_BRANCH: stackhpc/2023.1 | ||
steps: | ||
- name: Install Package | ||
uses: ConorMacBride/install-package@main | ||
with: | ||
apt: git unzip nodejs | ||
# If testing upgrade, checkout previous release, otherwise checkout current branch | ||
- name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config | ||
uses: actions/checkout@v4 | ||
with: | ||
ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }} | ||
submodules: true | ||
- name: Output Kayobe image | ||
id: kayobe_image | ||
run: | | ||
if ${{ inputs.upgrade }}; then | ||
kayobe_image=$PREVIOUS_KAYOBE_IMAGE | ||
else | ||
kayobe_image=$KAYOBE_IMAGE | ||
fi | ||
echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT | ||
- name: Make sure dockerd is running and test Docker | ||
run: | | ||
docker ps | ||
- name: Output image tag | ||
id: image_tag | ||
run: | | ||
echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT | ||
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag | ||
- name: Output image name | ||
id: image_name | ||
run: | | ||
if [ -z "${{ inputs.vm_image_override }}" ]; then | ||
echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT | ||
else | ||
echo image_name=${{ inputs.vm_image_override }} >> $GITHUB_OUTPUT | ||
fi | ||
- name: Install terraform | ||
uses: hashicorp/setup-terraform@v2 | ||
- name: Initialise terraform | ||
run: terraform init | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
- name: Generate SSH keypair | ||
run: ssh-keygen -f id_rsa -N '' | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
- name: Generate clouds.yaml | ||
run: | | ||
cat << EOF > clouds.yaml | ||
${{ secrets.CLOUDS_YAML }} | ||
EOF | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
- name: Generate terraform.tfvars | ||
run: | | ||
cat << EOF > terraform.tfvars | ||
ssh_public_key = "id_rsa.pub" | ||
ssh_username = "${{ env.SSH_USERNAME }}" | ||
aio_vm_interface = "${{ env.VM_INTERFACE }}" | ||
aio_vm_name = "${{ env.VM_NAME }}" | ||
aio_vm_image = "${{ env.VM_IMAGE }}" | ||
aio_vm_flavor = "${{ env.VM_FLAVOR }}" | ||
aio_vm_network = "${{ env.VM_NETWORK }}" | ||
aio_vm_subnet = "${{ env.VM_SUBNET }}" | ||
aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}" | ||
aio_vm_tags = ${{ env.VM_TAGS }} | ||
EOF | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
env: | ||
SSH_USERNAME: "${{ inputs.ssh_username }}" | ||
VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}" | ||
VM_IMAGE: ${{ steps.image_name.outputs.image_name }} | ||
VM_FLAVOR: ${{ inputs.vm_flavor }} | ||
VM_NETWORK: ${{ inputs.vm_network }} | ||
VM_SUBNET: ${{ inputs.vm_subnet }} | ||
VM_INTERFACE: ${{ inputs.vm_interface }} | ||
VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }} | ||
VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]' | ||
- name: Terraform Plan | ||
run: terraform plan | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
env: | ||
OS_CLOUD: ${{ inputs.OS_CLOUD }} | ||
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | ||
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | ||
- name: Terraform Apply | ||
id: tf_apply | ||
run: | | ||
for attempt in $(seq 5); do | ||
if terraform apply -auto-approve; then | ||
echo "Created infrastructure on attempt $attempt" | ||
exit 0 | ||
fi | ||
echo "Failed to create infrastructure on attempt $attempt" | ||
sleep 10 | ||
terraform destroy -auto-approve | ||
sleep 60 | ||
done | ||
echo "Failed to create infrastructure after $attempt attempts" | ||
exit 1 | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
env: | ||
OS_CLOUD: ${{ inputs.OS_CLOUD }} | ||
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | ||
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | ||
- name: Get Terraform outputs | ||
id: tf_outputs | ||
run: | | ||
terraform output -json | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
- name: Write Terraform outputs | ||
run: | | ||
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml | ||
${{ steps.tf_outputs.outputs.stdout }} | ||
EOF | ||
- name: Write Terraform network config | ||
run: | | ||
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml | ||
admin_oc_net_name: admin | ||
admin_cidr: "{{ access_cidr.value }}" | ||
admin_allocation_pool_start: 0.0.0.0 | ||
admin_allocation_pool_end: 0.0.0.0 | ||
admin_gateway: "{{ access_gw.value }}" | ||
admin_bootproto: dhcp | ||
admin_ips: | ||
controller0: "{{ access_ip_v4.value }}" | ||
EOF | ||
- name: Write Terraform network interface config | ||
run: | | ||
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces | ||
admin_interface: "{{ access_interface.value }}" | ||
EOF | ||
- name: Write all-in-one scenario config | ||
run: | | ||
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/zz-aio-scenario.yml | ||
--- | ||
os_distribution: ${{ env.OS_DISTRIBUTION }} | ||
os_release: "${{ env.OS_RELEASE }}" | ||
kolla_enable_ovn: ${{ env.ENABLE_OVN }} | ||
EOF | ||
env: | ||
ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }} | ||
OS_DISTRIBUTION: ${{ inputs.os_distribution }} | ||
OS_RELEASE: ${{ inputs.os_release }} | ||
# Use a heredoc to define a multiline string output | ||
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | ||
- name: Set SSH key output | ||
id: ssh_key | ||
run: | | ||
echo "ssh_key<<EOF" >> $GITHUB_OUTPUT | ||
cat terraform/aio/id_rsa >> $GITHUB_OUTPUT | ||
echo "EOF" >> $GITHUB_OUTPUT | ||
# The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image. | ||
- name: Pull previous Kayobe image | ||
run: | | ||
docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }} | ||
if: inputs.upgrade | ||
# The same tag may be reused (e.g. pr-123), so ensure we have the latest image. | ||
- name: Pull current Kayobe image | ||
run: | | ||
docker image pull $KAYOBE_IMAGE | ||
- name: Reboot | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml -e reboot_with_bootstrap_user=true | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Run growroot | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml' | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Host configure | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Service deploy | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Configure aio resources | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Change rabbit queues from HA to Quorum | ||
run: sed -i -e 's/om_enable_rabbitmq_high_availability: true/om_enable_rabbitmq_high_availability: false' -e 's/om_enable_rabbitmq_quorum_queues: false/om_enable_rabbitmq_quorum_queues: true' etc/kayobe/environments/ci-aio/kolla/globals.yml | ||
if: inputs.upgrade | ||
- name: Migrate RabbitMQ queues | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml --tags rabbit-queue-migration | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: inputs.upgrade | ||
# If testing upgrade, checkout the current release branch | ||
# Stash changes to tracked files, and set clean=false to avoid removing untracked files. | ||
- name: Stash config changes | ||
run: git stash | ||
if: inputs.upgrade | ||
- name: Checkout current release config | ||
uses: actions/checkout@v4 | ||
with: | ||
submodules: true | ||
clean: false | ||
if: inputs.upgrade | ||
- name: Pop stashed config changes | ||
run: git stash pop | ||
if: inputs.upgrade | ||
# Now begin upgrade | ||
- name: Host upgrade | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: inputs.upgrade | ||
- name: Host configure | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: inputs.upgrade | ||
- name: Service upgrade | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: inputs.upgrade | ||
- name: Ensure we have IP on breth1 to reach the instances | ||
# NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances | ||
# we can lose the IP address that allows us to connect to the instances. This playbook | ||
# also fixes that issue. | ||
run: | | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
${{ steps.kayobe_image.outputs.kayobe_image }} \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: inputs.upgrade | ||
- name: Tempest tests | ||
id: tempest | ||
run: | | ||
mkdir -p tempest-artifacts | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: StackHPC OpenStack tests | ||
id: stackhpc-openstack-tests | ||
continue-on-error: true | ||
run: | | ||
mkdir -p sot-results | ||
docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-v $(pwd)/sot-results:/stack/sot-results \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml' | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
- name: Collect diagnostic information | ||
id: diagnostics | ||
run: | | ||
mkdir -p diagnostics | ||
sudo -E docker run -t --rm \ | ||
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \ | ||
-v $(pwd)/diagnostics:/stack/diagnostics \ | ||
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \ | ||
$KAYOBE_IMAGE \ | ||
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml' | ||
env: | ||
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }} | ||
if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }} | ||
- name: Upload test result artifacts | ||
uses: actions/upload-artifact@v4 | ||
with: | ||
name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }} | ||
path: | | ||
diagnostics/ | ||
tempest-artifacts/ | ||
sot-results/ | ||
if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }} | ||
- name: Fail if any Tempest tests failed | ||
run: | | ||
test $(wc -l < tempest-artifacts/failed-tests) -lt 1 | ||
- name: Fail if any StackHPC OpenStack tests failed | ||
run: | | ||
echo "Some StackHPC OpenStack tests failed." | ||
echo "See HTML results artifact (sot-results) for details." | ||
exit 1 | ||
if: steps.stackhpc-openstack-tests.outcome == 'failure' | ||
- name: Destroy | ||
run: terraform destroy -auto-approve | ||
working-directory: ${{ github.workspace }}/terraform/aio | ||
env: | ||
OS_CLOUD: ${{ inputs.OS_CLOUD }} | ||
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }} | ||
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }} | ||
if: always() |