DRAFT Read replication and moving the API to a second database

.github/ISSUE_TEMPLATE/onboarding.md

@@ -80,6 +80,8 @@ Note: If you're not able to do any of these yourself, you're still responsible f
 - [ ] [Add as a form manager to the touchpoints recruitment intercept](https://touchpoints.app.cloud.gov/admin/forms/9412c559/permissions)
 
 **For engineers, also...**
+- [ ] Make sure you have a `login.gov` account and have logged into the FAC application at least once. 
+  - [ ] Then, add your email to the `readonly` list in [staffusers.json](../../backend/config/staffusers.json).
 - [ ] [Add as a member of the FAC group in New Relic](https://one.newrelic.com/admin-portal/organizations/users-list) (@GSA-TTS/fac-admins can do this)
 
 **For product leads/owners, also...**
@@ -88,5 +90,6 @@ Note: If you're not able to do any of these yourself, you're still responsible f
 - [ ] Also give them the `Maintainer` role in [the FAC-team team in GitHub](https://github.com/orgs/GSA-TTS/teams/fac-team/members).
 
 **For helpdesk, also...**
-- [ ] Add them to the list of staff users for [Django Admin](https://app.fac.gov/admin/users/staffuser/).
+- [ ] Make sure you have a `login.gov` account and have logged into the FAC application at least once. 
+  - [ ] Then, add your email to the `helpdesk` list in [staffusers.json](../../backend/config/staffusers.json).
 - [ ] Give them access to the [Help Desk](https://fac-gov.zendesk.com/admin/people/team/members) as a team member.

.github/workflows/deploy-application.yml

@@ -77,7 +77,7 @@ jobs:
           cf_password: ${{ secrets.CF_PASSWORD }}
           cf_org: gsa-tts-oros-fac
           cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.5 deploy_backup"
+          command: cf run-task gsa-fac -k 7G -m 3G --name deploy_backup --command "./fac-backup-util.sh v0.1.9 deploy_backup"
 
       - name: Deploy Preview to cloud.gov
         if: ${{ inputs.environment == 'preview' }}
@@ -103,15 +103,6 @@ jobs:
           cf_vars_file: backend/manifests/vars/vars-${{ env.space }}.yml
           command: bin/ops/deploy.sh
 
-      - name: Load historical data
-        uses: cloud-gov/cg-cli-tools@main
-        with:
-          cf_username: ${{ secrets.CF_USERNAME }}
-          cf_password: ${{ secrets.CF_PASSWORD }}
-          cf_org: gsa-tts-oros-fac
-          cf_space: ${{ env.space }}
-          command: cf run-task gsa-fac -k 6G -m 1G --name load_data --command "./load_data.sh"
-
       # This has to happen after an application deployment because the manifest (currently) is responsible
       # for binding the "logdrain service" to the "gsa-fac application". This also needs to be done
       # based on the suspicion that fluentbit cannot register the incoming logs when it is initially
@@ -132,5 +123,5 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ inputs.environment }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "check_tables"

.github/workflows/fac-api-scheduler.yml

@@ -0,0 +1,24 @@
+---
+name: API Refresh Schedule
+on:
+  schedule:
+    # Invoke at 12:30 EST
+    - cron: '30 4 * * *'
+  workflow_dispatch: null
+
+jobs:
+  scheduled-api-standup:
+    strategy:
+      fail-fast: false
+      matrix:
+        environment:
+          - name: dev
+          - name: staging
+          - name: production
+          - name: preview
+    uses: ./.github/workflows/fac-api-standup.yml
+    secrets: inherit
+    with:
+      environment: ${{ matrix.environment.name }}
+      util_version: "v0.1.9"
+      backup_operation: "rds_backup"

.github/workflows/fac-api-standup.yml

@@ -0,0 +1,54 @@
+---
+name: Standup the API
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      environment:
+        required: true
+        type: choice
+        options:
+          - 'dev'
+          - 'preview'
+          - 'staging'
+          - 'production'
+      util_version:
+        description: Version for fac backup utility to use (ex. vX.Y.Z)
+        required: true
+        type: string
+        default: "v0.1.9"
+      backup_operation:
+        description: Operation for fac-backup-utility
+        required: true
+        type: choice
+        options:
+          - 'rds_backup'
+jobs:
+  api-standup:
+    name: Standup API
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    env:
+      space: ${{ inputs.environment }}
+    steps:
+      - name: Backup FAC Database and refresh the api
+        uses: cloud-gov/cg-cli-tools@main
+        with:
+          cf_username: ${{ secrets.CF_USERNAME }}
+          cf_password: ${{ secrets.CF_PASSWORD }}
+          cf_org: gsa-tts-oros-fac
+          cf_space: ${{ env.space }}
+          command: cf run-task gsa-fac -k 2G -m 2G --name standup_api --command "./util/nightly_api_refresh.sh ${{ inputs.util_version }} ${{ inputs.backup_operation }}"
+          # command: cf run-task gsa-fac -k 2G -m 2G --name api_refresh --command "./util/nightly_api_refresh.sh v0.1.9 rds_backup"

.github/workflows/fac-backup-scheduler.yml

@@ -19,6 +19,5 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "scheduled_backup"
-

.github/workflows/fac-backup-util-scheduled.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 scheduled_backup
-# ./fac-backup-util.sh v0.1.8 daily_backup
+# ./fac-backup-util.sh v0.1.9 scheduled_backup
+# ./fac-backup-util.sh v0.1.9 daily_backup
 on:
   workflow_call:
     inputs:

.github/workflows/fac-backup-util.yml

@@ -1,8 +1,8 @@
 ---
 name: Backup the database with fac-backup-utility
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 initial_backup
-# ./fac-backup-util.sh v0.1.8 deploy_backup
+# ./fac-backup-util.sh v0.1.9 initial_backup
+# ./fac-backup-util.sh v0.1.9 deploy_backup
 on:
   workflow_dispatch:
     inputs:
@@ -18,6 +18,7 @@ on:
         description: Version for fac backup utility to use (ex. vX.Y.Z)
         required: true
         type: string
+        default: "v0.1.9"
       backup_operation:
         description: Operation for fac-backup-utility
         required: true

.github/workflows/fac-check-tables-scheduler.yml

@@ -20,5 +20,5 @@ jobs:
     secrets: inherit
     with:
       environment: ${{ matrix.environment.name }}
-      util_version: "v0.1.8"
+      util_version: "v0.1.9"
       backup_operation: "check_tables"

.github/workflows/fac-check-tables.yml

@@ -1,7 +1,7 @@
 ---
 name: Check existing tables in an environment
 ### Common Commands:
-# ./fac-backup-util.sh v0.1.8 check_tables
+# ./fac-backup-util.sh v0.1.9 check_tables
 on:
   workflow_dispatch:
     inputs:

backend/.gitignore

@@ -0,0 +1,2 @@
+*.tar.gz
+config.json

backend/.profile

@@ -1,20 +1,14 @@
 #!/bin/bash
 
-# Source everything; everything is now a function.
-# Remember: bash has no idea if a function exists,
-# so a typo in a function name will fail silently. Similarly,
-# bash has horrible scoping, so use of `local` in functions is
-# critical for cleanliness in the startup script.
+set +e
+
 source tools/util_startup.sh
-# This will choose the correct environment
-# for local envs (LOCAL or TESTING) and cloud.gov
 source tools/setup_env.sh
-source tools/api_teardown.sh
+source tools/curation_audit_tracking_disable.sh
+source tools/sling_bulk_export.sh
 source tools/migrate_app_tables.sh
-source tools/api_standup.sh
-source tools/run_collectstatic.sh
 source tools/seed_cog_baseline.sh
-source tools/materialized_views.sh
+source tools/sql_pre_post.sh
 
 #####
 # SETUP THE CGOV ENVIRONMENT
@@ -24,37 +18,37 @@ gonogo "setup_env"
 if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
 
     #####
-    # API TEARDOWN
-    # API has to be deprecated/removed before migration, because
-    # of tight coupling between schema/views and the dissemination tables
-    api_teardown
-    gonogo "api_teardown"
+    # SQL PRE
+    # We have SQL that we want to run before the migrations and sling are run.
+    # This tears down things that would conflict with migrations, etc.
+    sql_pre_fac_db
+    gonogo "sql_pre_fac_db"
+    curation_audit_tracking_disable
+    gonogo "curation_audit_tracking_disable"
 
     #####
     # MIGRATE APP TABLES
     migrate_app_tables
     gonogo "migrate_app_tables"
 
     #####
-    # API STANDUP
-    # Standup the API, which may depend on migration changes
-    api_standup
-    gonogo "api_standup"
-
-    #####
-    # COLLECT STATIC
-    # Do Django things with static files.
-    # run_collectstatic
-    # gonogo "run_collectstatic"
+    # SQL POST
+    # Rebuild the API and prepare the system for execution.
+    # Runs after migrations.
+    sql_post_fac_db
+    gonogo "sql_post_fac_db"
 
     #####
     # SEED COG/OVER TABLES
     # Setup tables for cog/over assignments
     seed_cog_baseline
     gonogo "seed_cog_baseline"
 
-    # materialized_views
-    # gonogo "materialized_views"
+    #####
+    # CREATE STAFF USERS
+    # Prepares staff users for Django admin
+    python manage.py create_staffusers
+    gonogo "create_staffusers"
 fi
 
 # Make psql usable by scripts, for debugging, etc.

backend/Dockerfile

@@ -18,6 +18,7 @@ RUN apt-get -yq update && \
         gcc \
         gnupg \
         gnupg2 \
+        jq \
         postgresql-client \
         wget
 

backend/Makefile

@@ -88,9 +88,9 @@ docker-lint:
 	docker compose run web bash -c 'flake8 && black --check . && bandit -c pyproject.toml -r . && mypy . && djlint .'
 
 ghcr-first-run:
-	docker compose -f docker-compose-web.yml run web python manage.py makemigrations
-	docker compose -f docker-compose-web.yml run web python manage.py migrate
-	docker compose -f docker-compose-web.yml run web python manage.py loaddata
+	-docker compose -f docker-compose-web.yml run web python manage.py makemigrations
+	-docker compose -f docker-compose-web.yml run web python manage.py migrate
+	-docker compose -f docker-compose-web.yml run web python manage.py loaddata
 
 # Run Django tests with docker
 ghcr-test:
@@ -109,15 +109,33 @@ ghcr-nctest:
 ghcr-lint:
 	docker compose -f docker-compose-web.yml run web bash -c 'flake8 && black --check . && bandit -c pyproject.toml -r . && python -m pip install types-pytz && mypy . && djlint .'
 
+remove-coverage:
+	-rm -f .coverage.*
+
+compose-down:
+	docker compose down
+
+remove-processes:
+	docker rm -f $$(docker ps -a -q)
+
+remove-volumes:
+	-docker volume rm $$(docker volume ls -q)  
+
+remove-dangling-volumes: remove-volumes
+	-docker volume rm $$(docker volume ls -q --filter dangling=true)
+
+system-prune:
+	-docker system prune -f
+
+volume-prune:
+	-docker volume prune -f
+
 docker-clean:
 	docker compose down
 	docker rm -f $(shell docker ps -a -q)
 	docker volume rm $(shell docker volume ls -q)
 
-docker-full-clean:
-	rm -f .coverage.*
-	docker compose down
-	docker rm -f $(docker ps -a -q)
-	docker volume rm $(docker volume ls -q)
-	docker system prune -f
-	docker volume prune -f
+# Proceed past errors.
+# We want all of these commands to run, even if one fails.
+# make -i docker-full-clean
+docker-full-clean: remove-coverage compose-down remove-processes remove-dangling-volumes system-prune volume-prune

backend/census_historical_migration/README.md

@@ -20,7 +20,7 @@ This is implemented as a Django app to leverage existing management commands and
 - fac_s3.py - Uploads folders or files to an S3 bucket.
 
 ```bash
-python manage.py fac_s3 gsa-fac-private-s3 --upload --src census_historical_migration/data
+python manage.py fac_s3 fac-private-s3 --upload --src census_historical_migration/data
 ```
 
 - csv_to_postgres.py - Inserts data into Postgres tables using the contents of the CSV files in the S3 bucket. The first row of each file is assumed to have the column names (we convert to lowercase). The name of the table is determined by examining the name of the file. The sample source files do not have delimters for empty fields at the end of a line - so we assume these are nulls.
@@ -44,16 +44,16 @@ python manage.py csv_to_postgres --clean True
 1.  Download test Census data from https://drive.google.com/drive/folders/1TY-7yWsMd8DsVEXvwrEe_oWW1iR2sGoy into census_historical_migration/data folder.
     NOTE: Never check in the census_historical_migration/data folder into GitHub.
 
-2.  In the FAC/backend folder, run the following to load CSV files from census_historical_migration/data folder into gsa-fac-private-s3 bucket.
+2.  In the FAC/backend folder, run the following to load CSV files from census_historical_migration/data folder into fac-private-s3 bucket.
 
 ```bash
 docker compose run --rm web python manage.py fac_s3 \
-  gsa-fac-private-s3 \
+  fac-private-s3 \
   --upload \
   --src census_historical_migration/data
 ```
 
-3.  In the FAC/backend folder, run the following to read the CSV files from gsa-fac-private-s3 bucket and load into Postgres.
+3.  In the FAC/backend folder, run the following to read the CSV files from fac-private-s3 bucket and load into Postgres.
 
 ```bash
 docker compose run --rm web python manage.py \

backend/census_historical_migration/throwaway_scripts/reprocess_migration_cli_commands.py

@@ -1,7 +1,7 @@
 import argparse
 import time
 
-from util import (
+from census_historical_migration.throwaway_scripts.util import (
     trigger_migration_workflow,
 )
 import subprocess  # nosec

backend/census_historical_migration/throwaway_scripts/start_process_cli_commands.py

@@ -2,7 +2,9 @@
 import subprocess  # nosec
 import time
 
-from util import trigger_migration_workflow
+from census_historical_migration.throwaway_scripts.util import (
+    trigger_migration_workflow,
+)
 
 # This throwaway script spits out code that can be
 # copy-pasted into a bash script, or directly into the command line.

backend/config/db_url.py

@@ -1,12 +1,10 @@
 from django.core.exceptions import ImproperlyConfigured
 
 
-def get_db_url_from_vcap_services(
-    vcap,
-):
+def get_db_url_from_vcap_services(vcap, db_instance_name="fac-db"):
     database_url = None
     for db_service in vcap.get("aws-rds", []):
-        if db_service.get("instance_name") == "fac-db":
+        if db_service.get("instance_name") == db_instance_name:
             database_url = db_service["credentials"]["uri"]
             break