Adding restore for database

DFE-Digital · Nov 13, 2024 · 9571d6a · 9571d6a
1 parent 5893c95
commit 9571d6a
Show file tree

Hide file tree

Showing 5 changed files with 226 additions and 32 deletions.
diff --git a/.github/workflows/backup_production_database.yml b/.github/workflows/backup_production_database.yml
@@ -1,18 +1,88 @@
-name: Production DB nightly backup
+name: Backup database to Azure storage
+
 on:
   workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to backup
+        required: true
+        default: staging
+        type: choice
+        options:
+          - staging
+          - migration
+          - production
+      backup-file:
+        description: |
+          Backup file name (without extension). Default is npq_[env]_adhoc_YYYY-MM-DD. Set it explicitly when backing up a point-in-time (PTR) server. (Optional)
+        required: false
+        type: string
+        default: default
+      db-server:
+        description: |
+          Name of the database server. Default is the live server. When backing up a point-in-time (PTR) server, use the full name of the PTR server. (Optional)
+
   schedule:
     - cron: "30 3 * * *" # 03:30 UTC
 
+env:
+  SERVICE_NAME: npq-registration
+  SERVICE_SHORT: cpdnpq
+  TF_VARS_PATH: terraform/application/config
+
 jobs:
-  backup-production:
-    runs-on: ubuntu-20.04
-    environment: production
+  backup:
+    name: Backup database
+    runs-on: ubuntu-latest
+    environment:
+      name: ${{ inputs.environment || 'production' }}
+    env:
+      DEPLOY_ENV: ${{ inputs.environment || 'production'  }}
+      BACKUP_FILE: ${{ inputs.backup-file || 'schedule'  }}
+
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+
+      - uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Set environment variables
+        run: |
+          source global_config/${DEPLOY_ENV}.sh
+          tf_vars_file=${TF_VARS_PATH}/${DEPLOY_ENV}.tfvars.json
+          echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+          echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+          TODAY=$(date +"%F")
+          echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+          if [ "${BACKUP_FILE}" == "schedule" ]; then
+            BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}
+          elif [ "${BACKUP_FILE}" == "default" ]; then
+            BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_adhoc_${TODAY}
+          else
+            BACKUP_FILE=${BACKUP_FILE}
+          fi
+          echo "BACKUP_FILE=${BACKUP_FILE}" >> $GITHUB_ENV
+          echo "KEYVAULT_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-inf-kv" >> $GITHUB_ENV
+
+      - name: Fetch secrets from key vault
+        uses: azure/CLI@v2
+        id: key-vault-secrets
+        with:
+          inlineScript: |
+            SLACK_WEBHOOK=$(az keyvault secret show --name "SLACK-WEBHOOK" --vault-name ${KEYVAULT_NAME} --query "value" -o tsv)
+            echo "::add-mask::$SLACK_WEBHOOK"
+            echo "SLACK_WEBHOOK=$SLACK_WEBHOOK" >> $GITHUB_OUTPUT
 
-      - name: Backup and upload database
-        uses: ./.github/actions/backup-and-upload-database
+      - name: Backup ${{ env.DEPLOY_ENV }} postgres
+        uses: DFE-Digital/github-actions/backup-postgres@master
         with:
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${{ env.DEPLOY_ENV }}-web
+          cluster: ${{ env.CLUSTER }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          backup-file: ${{ env.BACKUP_FILE }}.sql
+          db-server-name: ${{ inputs.db-server }}
+          slack-webhook: ${{ steps.key-vault-secrets.outputs.SLACK_WEBHOOK }}
diff --git a/.github/workflows/restore_azure_database.yml b/.github/workflows/restore_azure_database.yml
@@ -0,0 +1,72 @@
+name: Restore database from Azure storage
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: Environment to restore
+        required: true
+        default: development
+        type: choice
+        options:
+          - separation
+          - migration
+          - staging
+          - production
+      confirm-production:
+        description: Must be set to true if restoring production
+        required: true
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. cpdnpq_prod_2024-08-09.sql.gz. The default value is today's scheduled backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: npq-registration
+  SERVICE_SHORT: cpdnpq
+  TF_VARS_PATH: terraform/application/config
+
+jobs:
+  restore:
+    name: Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' ) }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
+    steps:
+      - uses: actions/checkout@v4
+        name: Checkout
+
+      - name: Set environment variables
+        run: |
+          source global_config/${{ inputs.environment }}.sh
+          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
+          tf_vars_file=${{ env.TF_VARS_PATH }}/${CONFIG}/${{ inputs.environment }}.tfvars.json
+          echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+          echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+          echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+          TODAY=$(date +"%F")
+          echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+          if [ "${{ inputs.backup-file }}" != "" ]; then
+            BACKUP_FILE=${{ inputs.backup-file }}
+          else
+            BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+          fi
+          echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
+
+      - name: Restore ${{ inputs.environment }} postgres
+        uses: DFE-Digital/github-actions/restore-postgres-backup@master
+        with:
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${CONFIG}-web
+          cluster: ${{ env.CLUSTER }}
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          backup-file: ${{ env.BACKUP_FILE }}
diff --git a/.github/workflows/restore_snapshot_database.yml b/.github/workflows/restore_snapshot_database.yml
@@ -1,25 +1,72 @@
-name: Restore Snapshot DB from production DB
+name: Restore database from Azure storage
+
 on:
   workflow_dispatch:
     inputs:
       environment:
-        description: GitHub environment to backup and restore
+        description: Environment to restore
+        required: true
+        default: development
         type: choice
-        default: production
         options:
+          - separation
+          - migration
+          - staging
           - production
+      confirm-production:
+        description: Must be set to true if restoring production
         required: true
+        default: "false"
+        type: choice
+        options:
+          - "false"
+          - "true"
+      backup-file:
+        description: Name of the backup file in Azure storage. e.g. cpdnpq_prod_2024-08-09.sql.gz. The default value is today's scheduled backup.
+        type: string
+        required: false
+
+env:
+  SERVICE_NAME: npq-registration
+  SERVICE_SHORT: cpdnpq
+  TF_VARS_PATH: terraform/application/config
 
 jobs:
-  backup-and-restore-production:
-    runs-on: ubuntu-20.04
-    environment: production
+  restore:
+    name: Restore AKS Database
+    if: ${{ inputs.environment != 'production' || (inputs.environment == 'production' && github.event.inputs.confirm-production == 'true' ) }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    concurrency: deploy_${{ inputs.environment }}
+
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        name: Checkout
+
+      - name: Set environment variables
+        run: |
+          source global_config/${{ inputs.environment }}.sh
+          echo "CONFIG=${CONFIG}" >> $GITHUB_ENV
+          tf_vars_file=${{ env.TF_VARS_PATH }}/${CONFIG}/${{ inputs.environment }}.tfvars.json
+          echo "CLUSTER=$(jq -r '.cluster' ${tf_vars_file})" >> $GITHUB_ENV
+          echo "RESOURCE_GROUP_NAME=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-rg" >> $GITHUB_ENV
+          echo "STORAGE_ACCOUNT_NAME=${AZURE_RESOURCE_PREFIX}${SERVICE_SHORT}dbbkp${CONFIG_SHORT}sa" >> $GITHUB_ENV
+          echo "DB_SERVER=${AZURE_RESOURCE_PREFIX}-${SERVICE_SHORT}-${CONFIG_SHORT}-pg" >> $GITHUB_ENV
+          TODAY=$(date +"%F")
+          echo "BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql" >> $GITHUB_ENV
+          if [ "${{ inputs.backup-file }}" != "" ]; then
+            BACKUP_FILE=${{ inputs.backup-file }}
+          else
+            BACKUP_FILE=${SERVICE_SHORT}_${CONFIG_SHORT}_${TODAY}.sql.gz
+          fi
+          echo "BACKUP_FILE=$BACKUP_FILE" >> $GITHUB_ENV
 
-      - name: Backup and restore snapshot
-        uses: ./.github/actions/backup-and-restore-snapshot-database
+      - name: Restore ${{ inputs.environment }} postgres
+        uses: DFE-Digital/github-actions/restore-postgres-backup@master
         with:
-          environment: production
+          storage-account: ${{ env.STORAGE_ACCOUNT_NAME }}
+          resource-group: ${{ env.RESOURCE_GROUP_NAME }}
+          app-name: ${{ env.SERVICE_NAME }}-${CONFIG}-web
+          cluster: ${{ env.CLUSTER }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+          backup-file: ${{ env.BACKUP_FILE }}
diff --git a/terraform/application/.terraform.lock.hcl b/terraform/application/.terraform.lock.hcl
diff --git a/terraform/application/config/migration.tfvars.json b/terraform/application/config/migration.tfvars.json
@@ -1,6 +1,7 @@
 {
     "cluster": "production",
     "namespace": "cpd-production",
+    "enable_postgres_backup_storage" : true,
     "environment": "migration",
     "deploy_redis_cache": true,
     "redis_cache_family": "C",