From 93116b49d79477a09821c966059f5dbf362f5f8b Mon Sep 17 00:00:00 2001 From: harishsurf Date: Tue, 9 Jul 2024 17:19:18 +0000 Subject: [PATCH] Add CI job for testing db upgrade from postgres to sqlite --- .github/workflows/jobs.yml | 78 +++++++++++++++++++ .../tasks/upgrade-config-vars.yaml | 13 +++- .../roles/mirror_appliance/tasks/upgrade.yaml | 6 ++ 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/.github/workflows/jobs.yml b/.github/workflows/jobs.yml index f6d9299..41d6310 100644 --- a/.github/workflows/jobs.yml +++ b/.github/workflows/jobs.yml @@ -4,6 +4,9 @@ on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' + pull_request: + branches: + - "*" pull_request_target: types: [labeled] branches: @@ -184,6 +187,42 @@ jobs: - name: Uninstall Registry run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa + - name: Download old mirror-registry tarball that runs quay with postgres + run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz + + - name: Create extraction directory for old mirror-registry binary + run: mkdir -p ./mirror-registry-postgres + + - name: Extract tarball into the folder + run: tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres + + - name: Install postgres backed quay registry + run: ./mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa + + - name: Podman login to quay registry + run: podman login -u init -p password quay:8443 --tls-verify=false + + - name: Pull busybox image from Docker Hub + run: podman pull docker.io/library/busybox + + - name: Tag busybox image for Quay + run: podman tag docker.io/library/busybox quay:8443/init/busybox:latest + + - name: Push busybox image to Quay + run: podman push quay:8443/init/busybox:latest --tls-verify=false + + - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite + run: ./mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa + + - name: Pull already pushed busybox image from quay registry + run: podman pull quay:8443/init/busybox:latest + + - name: Verify busybox image is pulled successfully + run: podman images | grep -q quay:8443/init/busybox:latest + + - name: Uninstall Registry + run: ./mirror-registry uninstall -u jonathan -H quay --autoApprove -v -k /home/runner/.ssh/id_rsa + - name: Terraform Destroy run: terraform destroy --auto-approve shell: bash @@ -292,6 +331,45 @@ jobs: - name: Uninstall Quay run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v' + - name: Download old mirror-registry tarball that runs quay with postgres + run: wget -O mirror-registry-postgres.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.10/mirror-registry-online.tar.gz + + - name: SCP old tarball to VM + run: scp mirror-registry-postgres.tar.gz jonathan@quay:~/mirror-registry-postgres.tar.gz + + - name: Create extraction directory for old mirror-registry binary + run: ssh jonathan@quay 'mkdir -p ./mirror-registry-postgres' + + - name: Extract tarball into the folder + run: ssh jonathan@quay 'tar -xzf mirror-registry-postgres.tar.gz -C ./mirror-registry-postgres' + + - name: Install postgres backed quay registry + run: ssh jonathan@quay './mirror-registry-postgres/mirror-registry install -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa' + + - name: Podman login to quay registry + run: ssh jonathan@quay 'podman login -u init -p password quay:8443 --tls-verify=false' + + - name: Pull busybox image from Docker Hub + run: ssh jonathan@quay 'podman pull docker.io/library/busybox' + + - name: Tag busybox image for Quay + run: ssh jonathan@quay 'podman tag docker.io/library/busybox quay:8443/init/busybox:latest' + + - name: Push busybox image to Quay + run: ssh jonathan@quay 'podman push quay:8443/init/busybox:latest --tls-verify=false' + + - name: Use latest binary to test upgrade cmd to ensure db migration from old postgres to sqlite + run: ssh jonathan@quay './mirror-registry upgrade -u jonathan -r /home/jonathan/quay-install -H quay -v --initPassword password -k /home/runner/.ssh/id_rsa' + + - name: Pull already pushed busybox image from quay registry + run: ssh jonathan@quay 'podman pull quay:8443/init/busybox:latest' + + - name: Verify busybox image was pulled successfully + run: ssh jonathan@quay 'podman images | grep -q quay:8443/init/busybox:latest' + + - name: Uninstall Quay + run: ssh jonathan@quay './mirror-registry uninstall --autoApprove -v' + - name: Terraform Destroy run: terraform destroy --auto-approve shell: bash diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml index 9dfff3d..596c58c 100644 --- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml +++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade-config-vars.yaml @@ -11,8 +11,17 @@ ansible.builtin.set_fact: quay_config_file: "{{ remote_yaml_file['content'] | b64decode | from_yaml }}" -- name: Set facts for the pre-existing secrets only if they are a string and not a jinja2 variable in the config.yaml. +- name: Set facts for the existing redis secrets only if they are a string and not a jinja2 variable in the config.yaml. ansible.builtin.set_fact: REDIS_PASSWORD : "{{ quay_config_file['USER_EVENTS_REDIS']['password'] }}" + when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string + +- name: Check if quay-postgres container is running + command: podman ps -q -f name=quay-postgres + register: postgres_container_status + changed_when: false + +- name: Set facts for existing postgres secrets only if they are a string and not a jinja2 variable in the config.yaml. + ansible.builtin.set_fact: PGDB_PASSWORD : "{{ quay_config_file['DB_URI'].split('@')[0].split(':')[2] }}" - when: quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['USER_EVENTS_REDIS']['password'] is string and quay_config_file['DB_URI'] is string \ No newline at end of file + when: postgres_container_status.stdout != "" and quay_config_file['DATABASE_SECRET_KEY'] is string and quay_config_file['DB_URI'] is string \ No newline at end of file diff --git a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml index 2e4c742..cd65231 100755 --- a/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml +++ b/ansible-runner/context/app/project/roles/mirror_appliance/tasks/upgrade.yaml @@ -10,8 +10,14 @@ - name: Autodetect existing Secrets in config.yaml include_tasks: upgrade-config-vars.yaml +- name: Check if quay-postgres container is running + command: podman ps -q -f name=quay-postgres + register: postgres_container_status + changed_when: false + - name: Migrate postgres db to sqlite for Quay include_tasks: migrate.yaml + when: postgres_container_status.stdout != "" - name: Autodetect Image Archive include_tasks: autodetect-image-archive.yaml