Merge pull request #3334 from DFE-Digital/2189-gse-plan-for-dr-test

Add new postgres backup and restore workflows

.github/workflows/db_backup.yml

@@ -1,24 +1,88 @@
 name: Backup Database
+
 on:
   workflow_dispatch:
-  schedule: # 01:00 UTC
-    - cron: "0 1 * * *"
+    inputs:
+      environment:
+        description: Environment to backup
+        required: true
+        default: development
+        type: choice
+        options:
+        - development
+        - staging
+        - production
+      backup-file:
+        description: |
+          Backup file name (without extension). Default is gse_[env]_adhoc_YYYY-MM-DD. Set it explicitly when backing up a point-in-time (PTR) server. (Optional)
+        required: false
+        type: string
+        default: default
+      db-server:
+        description: |
+          Name of the database server. Default is the live server. When backing up a point-in-time (PTR) server, use the full name of the PTR server. (Optional)
+
+  schedule:
+    - cron: "0 1 * * *" # 01:00 UTC
+
+env:
+  SERVICE_NAME: get-school-experience
+  SERVICE_SHORT: gse
+  TF_VARS_PATH: terraform/aks/config
+
 jobs:
   backup:
     name: Backup Database
     runs-on: ubuntu-latest
-    strategy:
-      max-parallel: 1
-      matrix:
-        environment: [development,  staging, production]
     environment:
-       name: ${{matrix.environment}}
-    concurrency: ${{matrix.environment}}_${{github.event.number}}
+      name: ${{ inputs.environment || 'production' }}
+    env:
+      DEPLOY_ENV: ${{ inputs.environment || 'production'  }}
+      BACKUP_FILE: ${{ inputs.backup-file || 'schedule'  }}
+
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-      - name: Backup database
-        uses: ./.github/workflows/actions/database-backup
-        with:
-          environment: ${{matrix.environment}}
-          azure_credentials: ${{ secrets.AZURE_CREDENTIALS }}
+    - uses: actions/checkout@v4
+
+    - uses: azure/login@v2
+      with:
+        creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+    - name: Set environment variables
+      run: |
+        source global_config/${DEPLOY_ENV}.sh
+        tf_vars_file=${TF_VARS_PATH}/${DEPLOY_ENV}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        if [ "${BACKUP_FILE}" == "schedule" ]; then
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}
+        elif [ "${BACKUP_FILE}" == "default" ]; then
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_adhoc_${TODAY}
+        else
+          BACKUP_FILE=${BACKUP_FILE}
+        fi
+        echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV
+        echo "KEYVAULT_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV
+
+    - name: Fetch secrets from key vault
+      uses: azure/CLI@v2
+      id: key-vault-secrets
+      with:
+        inlineScript: |
+          SLACK_WEBHOOK=$(az keyvault secret show --name "SLACK-WEBHOOK" --vault-name ${KEYVAULT_NAME} --query "value" -o tsv)
+          echo "::add-mask::$SLACK_WEBHOOK"
+          echo "SLACK_WEBHOOK=$SLACK_WEBHOOK" >> $GITHUB_OUTPUT
+
+    - name: Backup ${{ env.DEPLOY_ENV }} postgres
+      uses: DFE-Digital/github-actions/backup-postgres@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: get-school-experience-${{ env.DEPLOY_ENV }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+        backup-file: ${{ env.BACKUP_FILE }}.sql
+        db-server-name: ${{ inputs.db-server }}
+        slack-webhook: ${{ steps.key-vault-secrets.outputs.SLACK_WEBHOOK }}

.github/workflows/postgres-ptr.yml

@@ -0,0 +1,70 @@
+name: Restore database from point in time to new database server
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: development
+        type: choice
+        options:
+        - development
+        - staging
+        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      restore-time:
+        description: Restore point in time in UTC. e.g. 2024-07-24T06:00:00
+        type: string
+        required: true
+      new-db-server:
+        description: Name of the new database server. Default is <original-server-name>-ptr.
+        type: string
+
+env:
+  SERVICE_SHORT: gse
+  TF_VARS_PATH: terraform/aks/config
+
+jobs:
+  ptr-restore:
+    name: PTR Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${TF_VARS_PATH}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+
+        DB_SERVER="${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg"
+        if [[ -n "${{ inputs.new-db-server }}" ]]; then
+          NEW_DB_SERVER="${{ inputs.new-db-server }}"
+        else
+          NEW_DB_SERVER="${DB_SERVER}-ptr"
+        fi
+        echo "DB_SERVER=${DB_SERVER}" >> $GITHUB_ENV
+        echo "NEW_DB_SERVER=${NEW_DB_SERVER}" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/ptr-postgres@master
+      with:
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        source-server: ${{ env.DB_SERVER }}
+        new-server: ${{ env.NEW_DB_SERVER }}
+        restore-time: ${{ inputs.restore-time }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS}}

.github/workflows/postgres-restore.yml

@@ -0,0 +1,70 @@
+name: Restore database from Azure storage
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: development
+        type: choice
+        options:
+        - development
+        - staging
+        - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: 'false'
+        type: choice
+        options:
+        - 'false'
+        - 'true'
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. gse_prod_2024-08-09.sql.gz. The default value is today's scheduled backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: get-school-experience
+  SERVICE_SHORT: gse
+  TF_VARS_PATH: terraform/aks/config
+
+jobs:
+  restore:
+    name: Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && inputs.confirm-production == 'true' )  }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout
+
+    - name: Set environment variables
+      run: |
+        source global_config/${{ inputs.environment }}.sh
+        tf_vars_file=${{ env.TF_VARS_PATH }}/${{ inputs.environment }}.tfvars.json
+        echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+        echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+        echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+        echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+        TODAY=$(date +"%F")
+        echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+        if [ "${{ inputs.backup-file }}" != "" ]; then
+          BACKUP_FILE=${{ inputs.backup-file }}
+        else
+          BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+        fi
+        echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+
+    - name: Restore ${{ inputs.environment }} postgres
+      uses: DFE-Digital/github-actions/restore-postgres-backup@master
+      with:
+        storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+        resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+        app-name: ${{ env.SERVICE_NAME }}-${{ inputs.environment }}
+        cluster: ${{ env.CLUSTER }}
+        azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+        backup-file: ${{ env.BACKUP_FILE }}