Added paas to aks backup and restore

WHY: IT is essential that the db is restored to aks HOW: By adding the backup an restore function to workflow
DFE-Digital · Sep 26, 2023 · ad39602 · ad39602
1 parent 490735b
commit ad39602
Show file tree

Hide file tree

Showing 8 changed files with 510 additions and 0 deletions.
diff --git a/.github/workflows/paas_to_aks_db_backup_and_restore_manual.yml b/.github/workflows/paas_to_aks_db_backup_and_restore_manual.yml
@@ -0,0 +1,172 @@
+name: Backup and restore Postgres DB from PAAS to AKS
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment
+        type: choice
+        options:
+          - development
+          - staging
+          - production
+
+env:
+  BACKUP_ARTIFACT_NAME: ${{ inputs.environment }}-backup
+
+jobs:
+  backup:
+    name: Backup from PAAS
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}_aks
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - uses: DFE-Digital/github-actions/install-postgres-client@master
+
+      - name: Set AKS environment name
+        id: set_aks_env_name
+        run: |
+          case "${{ inputs.environment }}" in
+          development)
+            echo "ENVIRONMENT_ABR=dev" >> $GITHUB_ENV
+            echo "TFVARS_NAME=dev" >> $GITHUB_ENV
+            echo "KEY_VAULT_NAME=s189t01-gse-dv-inf-kv" >> $GITHUB_OUTPUT
+            ;;
+          staging)
+            echo "ENVIRONMENT_ABR=staging" >> $GITHUB_ENV
+            echo "TFVARS_NAME=staging" >> $GITHUB_ENV
+            echo "KEY_VAULT_NAME=s189t01-gse-stg-inf-kv" >> $GITHUB_OUTPUT
+            ;;
+          production)
+            echo "ENVIRONMENT_ABR=prod" >> $GITHUB_ENV
+            echo "TFVARS_NAME=production" >> $GITHUB_ENV
+            echo "KEY_VAULT_NAME=s189p01-gse-pd-inf-kv" >> $GITHUB_OUTPUT
+            ;;
+          *)
+            echo "unknown cluster"
+            ;;
+          esac
+
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          tf_vars_file=terraform/aks/config/${{ inputs.environment }}.tfvars.json
+          echo "KEY_VAULT_NAME=$(jq -r '.infra_key_vault_name' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "PAAS_SPACE=$(jq -r '.paas_space' ${tf_vars_file})" >> $GITHUB_ENV
+
+      - name: Retrieve Cloudfoundry credentials from KV
+        uses: azure/CLI@v1
+        id: fetch-cf-creds
+        with:
+          inlineScript: |
+            SECRET_VALUE=$(az keyvault secret show --name "PAAS-USERNAME" --vault-name "${{ env.KEY_VAULT_NAME}}" --query "value" -o tsv)
+            echo "::add-mask::$SECRET_VALUE"
+            echo "PAAS-USER=$SECRET_VALUE" >> $GITHUB_OUTPUT
+
+            SECRET_VALUE=$(az keyvault secret show --name "PAAS-PASSWORD" --vault-name "${{ env.KEY_VAULT_NAME}}" --query "value" -o tsv)
+            echo "::add-mask::$SECRET_VALUE"
+            echo "PAAS-PASSWORD=$SECRET_VALUE" >> $GITHUB_OUTPUT
+
+      - uses: DFE-Digital/github-actions/setup-cf-cli@master
+        with:
+          CF_USERNAME: ${{ steps.fetch-cf-creds.outputs.PAAS-USER }}
+          CF_PASSWORD: ${{ steps.fetch-cf-creds.outputs.PAAS-PASSWORD }}
+          CF_SPACE_NAME: ${{ env.PAAS_SPACE }}
+          INSTALL_CONDUIT: true
+
+      - name: Backup database
+        run: |
+          cf conduit school-experience-${{ env.ENVIRONMENT_ABR }}-pg-common-svc -- pg_dump -E utf8 --clean --compress=1 --if-exists --no-owner --no-privileges --verbose -f backup.sql.gz
+
+      - name: Upload backup
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.BACKUP_ARTIFACT_NAME }}
+          path: backup.sql.gz
+          retention-days: 1
+
+
+
+  restore:
+    name: Restore to AKS
+    runs-on: ubuntu-latest
+    needs: backup
+
+    environment: ${{ inputs.environment }}_aks
+    env:
+      KEY_VAULT_NAME: ${{ needs.backup.outputs.KEY_VAULT_NAME }}
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: Azure/login@v1
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set environment variables
+        shell: bash
+        run: |
+          tf_vars_file=terraform/aks/config/${{ inputs.environment }}.tfvars.json
+
+      - run: |
+          test_cluster_rg=s189t01-tsc-ts-rg
+          test_cluster_name=s189t01-tsc-test-aks
+
+          prod_cluster_rg=s189p01-tsc-pd-rg
+          prod_cluster_name=s189p01-tsc-production-aks
+
+          case "${{ inputs.environment }}" in
+          development)
+            echo "in development with  rg = $test_cluster_rg clustername = $test_cluster_name" >> $GITHUB_ENV
+            echo "cluster_rg=$test_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$test_cluster_name" >> $GITHUB_ENV
+            echo "app_name=get-school-experience-development" >> $GITHUB_ENV
+            echo "key_vault_name=s189t01-gse-dv-inf-kv" >> $GITHUB_ENV
+            ;;
+          staging)
+            echo "cluster_rg=$test_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$test_cluster_name" >> $GITHUB_ENV
+            echo "app_name=get-school-experience-staging" >> $GITHUB_ENV
+            echo "key_vault_name=s189t01-gse-stg-inf-kv" >> $GITHUB_ENV
+            ;;
+          production)
+            echo "cluster_rg=$prod_cluster_rg" >> $GITHUB_ENV
+            echo "cluster_name=$prod_cluster_name" >> $GITHUB_ENV
+            echo "app_name=get-school-experience-production" >> $GITHUB_ENV
+            echo "key_vault_name=s189p01-gse-pd-inf-kv" >> $GITHUB_ENV
+            ;;
+          *)
+            echo "unknown cluster"
+            ;;
+          esac
+
+      - uses: azure/setup-kubectl@v3
+
+      - run: |
+          az aks get-credentials -g ${{ env.cluster_rg }} -n ${{ env.cluster_name }}
+          make bin/konduit.sh
+
+      - name: Download backup
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.BACKUP_ARTIFACT_NAME }}
+
+      - name: Restore database
+        run: bin/konduit.sh -i backup.sql.gz -c -k ${{ env.key_vault_name }} -d gse-${{ inputs.environment }} get-school-experience-${{ inputs.environment }}  -- psql
+
+
+      - name: Remove PaaS event triggers
+        shell: bash
+        run: |
+          bin/konduit.sh  -k ${{ env.key_vault_name }} -d gse-${{ inputs.environment }} get-school-experience-${{ inputs.environment }}  -- psql -c 'drop event trigger forbid_ddl_reader; drop event trigger make_readable; drop event trigger reassign_owned;'
+
+      - uses: geekyeggo/delete-artifact@v2
+        with:
+          name: ${{ env.BACKUP_ARTIFACT_NAME }}
diff --git a/Makefile b/Makefile
@@ -160,6 +160,10 @@ terraform-apply: terraform-init
 terraform-apply-aks: terraform-init-aks
 	terraform -chdir=terraform/aks apply -var-file "config/${CONFIG}.tfvars.json" ${AUTO_APPROVE}
 
+bin/konduit.sh:
+	curl -s https://raw.githubusercontent.com/DFE-Digital/teacher-services-cloud/main/scripts/konduit.sh -o bin/konduit.sh \
+		&& chmod +x bin/konduit.sh
+
 terraform-destroy: terraform-init
 	terraform -chdir=terraform/paas destroy -var-file=${DEPLOY_ENV}.env.tfvars ${AUTO_APPROVE}