2023-12-08 | MAIN --> PROD | DEV (8545813) --> STAGING

backend/.profile

@@ -1,65 +1,64 @@
 #!/bin/bash
-set -e
 
-export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
+# Source everything; everything is now a function.
+# Remember: bash has no idea if a function exists, 
+# so a typo in a function name will fail silently. Similarly,
+# bash has horrible scoping, so use of `local` in functions is 
+# critical for cleanliness in the startup script.
+source tools/util_startup.sh
+# This will choose the correct environment
+# for local envs (LOCAL or TESTING) and cloud.gov
+source tools/setup_env.sh
+source tools/migrate_historic_tables.sh
+source tools/api_teardown.sh
+source tools/migrate_app_tables.sh
+source tools/api_standup.sh
+source tools/run_collectstatic.sh
+source tools/seed_cog_baseline.sh
 
-export https_proxy="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.uri")"
-export smtp_proxy_domain="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.domain")"
-export smtp_proxy_port="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.port")"
 
-S3_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")"
-S3_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")"
-S3_PRIVATE_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")"
-S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")"
-export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal"
+if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
 
-# Grab the New Relic license key from the newrelic-creds user-provided service instance
-export NEW_RELIC_LICENSE_KEY="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "newrelic-creds" ".[][] | select(.name == \$service_name) | .credentials.NEW_RELIC_LICENSE_KEY")"
+    #####
+    # SETUP THE CGOV ENVIRONMENT
+    setup_env
+    gonogo "setup_env"
 
-# Set the application name for New Relic telemetry.
-export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)-$(echo "$VCAP_APPLICATION" | jq -r .space_name)"
+    #####
+    # MIGRATE HISTORICAL TABLES
+    # Migrate the historic tables first.
+    migrate_historic_tables
+    gonogo "migrate_historic_tables"
 
-# Set the environment name for New Relic telemetry.
-export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)"
+    #####
+    # API TEARDOWN
+    # API has to be deprecated/removed before migration, because
+    # of tight coupling between schema/views and the dissemination tables
+    api_teardown
+    gonogo "api_teardown"
 
-# Set Agent logging to stdout to be captured by CF Logs
-export NEW_RELIC_LOG=stdout
+    #####
+    # MIGRATE APP TABLES
+    migrate_app_tables
+    gonogo "migrate_app_tables"
 
-# Logging level, (critical, error, warning, info and debug). Default to info
-export NEW_RELIC_LOG_LEVEL=info
+    #####
+    # API STANDUP
+    # Standup the API, which may depend on migration changes
+    api_standup
+    gonogo "api_standup"
 
-# https://docs.newrelic.com/docs/security/security-privacy/compliance/fedramp-compliant-endpoints/
-export NEW_RELIC_HOST="gov-collector.newrelic.com"
-# https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy
-export NEW_RELIC_PROXY_HOST="$https_proxy"
+    #####
+    # COLLECT STATIC
+    # Do Django things with static files.
+    run_collectstatic
+    gonogo "run_collectstatic"
 
-# We only want to run migrate and collecstatic for the first app instance, not
-# for additional app instances, so we gate all of this behind CF_INSTANCE_INDEX
-# being 0.
-if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then
-    echo 'Starting API schema deprecation' &&
-    python manage.py drop_deprecated_api_schema_and_views &&
-    echo 'Finished API schema deprecation' &&
-    echo 'Dropping API schema' &&
-	python manage.py drop_api_schema &&
-	echo 'Finished dropping API schema' &&
-    echo 'Starting API schema creation' &&
-    python manage.py create_api_schema &&
-    echo 'Finished API schema creation' &&
-    echo 'Starting migrate' &&
-    python manage.py migrate &&
-    python manage.py migrate --database census-to-gsafac-db &&
-    echo 'Finished migrate' &&
-    echo 'Starting API view creation' &&
-    python manage.py create_api_views &&
-    echo 'Finished view creation' &&
-    echo 'Starting collectstatic' &&
-    python manage.py collectstatic --noinput &&
-    echo 'Finished collectstatic' &&
-    echo 'Starting seed_cog_baseline' &&
-    python manage.py seed_cog_baseline &&
-    echo 'Finished seed_cog_baseline'
+    #####
+    # SEED COG/OVER TABLES
+    # Setup tables for cog/over assignments
+    seed_cog_baseline
+    gonogo "seed_cog_baseline"
 fi
 
 # Make psql usable by scripts, for debugging, etc.

backend/census_historical_migration/README.md

@@ -68,7 +68,7 @@ To migrate dbkeys for a given year with pagination:
 docker compose run --rm web python manage.py run_paginated_migration
   --year 2022 \
   --page_size 1000
-  --pages 1, 3, 4
+  --pages 1,3,4
 ```
 - `batchSize` and `pages` are optional. The script will use default values for these if they aren't provided.
 

backend/census_historical_migration/historic_data_loader.py

@@ -1,4 +1,4 @@
-from .models import ELECAUDITHEADER as Gen
+from .models import ELECAUDITHEADER as AuditHeader
 from .workbooklib.end_to_end_core import run_end_to_end
 
 from django.contrib.auth import get_user_model
@@ -13,7 +13,7 @@ def load_historic_data_for_year(audit_year, page_size, pages):
     result_log = {}
     total_count = error_count = 0
     user = create_or_get_user()
-    submissions_for_year = Gen.objects.filter(AUDITYEAR=audit_year).order_by(
+    submissions_for_year = AuditHeader.objects.filter(AUDITYEAR=audit_year).order_by(
         "ELECAUDITHEADERID"
     )
     paginator = Paginator(submissions_for_year, page_size)
@@ -27,16 +27,11 @@ def load_historic_data_for_year(audit_year, page_size, pages):
         )
 
         for submission in page.object_list:
-            dbkey = submission.DBKEY
             result = {"success": [], "errors": []}
+            # Migrate a single submission
+            run_end_to_end(user, submission, result)
 
-            try:
-                # Migrate a single submission
-                run_end_to_end(user, dbkey, audit_year, result)
-            except Exception as exc:
-                result["errors"].append(f"{exc}")
-
-            result_log[(audit_year, dbkey)] = result
+            result_log[(audit_year, submission.DBKEY)] = result
             total_count += 1
 
             if len(result["errors"]) > 0:

backend/census_historical_migration/management/commands/historic_data_migrator.py

@@ -1,62 +1,65 @@
+from django.core.management.base import BaseCommand
 import logging
 import sys
+from census_historical_migration.sac_general_lib.utils import (
+    normalize_year_string,
+)
+from census_historical_migration.workbooklib.excel_creation_utils import (
+    get_audit_header,
+)
 from census_historical_migration.historic_data_loader import create_or_get_user
-
-from config.settings import ENVIRONMENT
-from django.core.management.base import BaseCommand
 from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end
+from django.conf import settings
 
 logger = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
     def add_arguments(self, parser):
-        parser.add_argument("--dbkeys", type=str, required=False, default="")
-        parser.add_argument("--years", type=str, required=False, default="")
+        parser.add_argument(
+            "--dbkeys", type=str, required=False, default="177310,251020"
+        )
+        parser.add_argument("--years", type=str, required=False, default="22,22")
 
-    def handle(self, *args, **options):
-        dbkeys_str = options["dbkeys"]
-        years_str = options["years"]
+    def initiate_migration(self, dbkeys_str, years_str):
         dbkeys = dbkeys_str.split(",")
-        years = years_str.split(",")
 
-        if len(dbkeys) != len(years):
-            logger.error(
-                "Received {} dbkeys and {} years. Must be equal. Exiting.".format(
-                    len(dbkeys), len(years)
+        if years_str:
+            years = [normalize_year_string(year) for year in years_str.split(",")]
+            if len(dbkeys) != len(years):
+                logger.error(
+                    "Received {} dbkeys and {} years. Must be equal. Exiting.".format(
+                        len(dbkeys), len(years)
+                    )
                 )
-            )
-            sys.exit(-1)
-
-        lengths = [len(s) == 2 for s in years]
-        if dbkeys_str and years_str and (not all(lengths)):
-            logger.error("Years must be two digits. Exiting.")
-            sys.exit(-2)
+                sys.exit(-1)
 
         user = create_or_get_user()
 
-        defaults = [
-            (177310, 22),
-            (251020, 22),
-        ]
+        if dbkeys_str and years_str:
+            logger.info(
+                f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}"
+            )
+            for dbkey, year in zip(dbkeys, years):
+                logger.info("Running {}-{} end-to-end".format(dbkey, year))
+                result = {"success": [], "errors": []}
+                try:
+                    audit_header = get_audit_header(dbkey, year)
+                except Exception as e:
+                    logger.error(e)
+                    continue
+
+                run_end_to_end(user, audit_header, result)
+                logger.info(result)
 
-        if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]:
-            if dbkeys_str and years_str:
-                logger.info(
-                    f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}"
-                )
-                for dbkey, year in zip(dbkeys, years):
-                    result = {"success": [], "errors": []}
-                    run_end_to_end(user, dbkey, year, result)
-                    logger.info(result)
-            else:
-                for pair in defaults:
-                    logger.info("Running {}-{} end-to-end".format(pair[0], pair[1]))
-                    result = {"success": [], "errors": []}
-                    run_end_to_end(user, str(pair[0]), str(pair[1]), result)
-                    logger.info(result)
+    def handle(self, *args, **options):
+        dbkeys_str = options["dbkeys"]
+        years_str = options["years"]
+
+        if settings.ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]:
+            self.initiate_migration(dbkeys_str, years_str)
         else:
             logger.error(
-                "Cannot run end-to-end workbook generation in production. Exiting."
+                "Cannot run end-to-end historic data  migrator in production. Exiting."
             )
             sys.exit(-3)

backend/census_historical_migration/management/commands/historic_workbook_generator.py

@@ -1,3 +1,7 @@
+from census_historical_migration.workbooklib.excel_creation_utils import (
+    get_audit_header,
+)
+from census_historical_migration.sac_general_lib.utils import normalize_year_string
 from census_historical_migration.workbooklib.workbook_builder import (
     generate_workbook,
 )
@@ -44,8 +48,8 @@ def handle(self, *args, **options):  # noqa: C901
                 logger.info(e)
                 logger.info(f"Could not create directory {out_basedir}")
                 sys.exit()
-
-        outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{options["year"]}')
+        year = normalize_year_string(options["year"])
+        outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{year[-2:]}')
 
         if not os.path.exists(outdir):
             try:
@@ -56,11 +60,10 @@ def handle(self, *args, **options):  # noqa: C901
                 logger.info("could not create output directory. exiting.")
                 sys.exit()
 
+        audit_header = get_audit_header(options["dbkey"], year)
         json_test_tables = []
         for section, fun in sections_to_handlers.items():
-            (wb, api_json, _, filename) = generate_workbook(
-                fun, options["dbkey"], options["year"], section
-            )
+            (wb, api_json, _, filename) = generate_workbook(fun, audit_header, section)
             if wb:
                 wb_path = os.path.join(outdir, filename)
                 wb.save(wb_path)

backend/census_historical_migration/management/commands/run_paginated_migration.py

@@ -1,3 +1,4 @@
+from census_historical_migration.sac_general_lib.utils import normalize_year_string
 from ...historic_data_loader import load_historic_data_for_year
 
 from django.core.management.base import BaseCommand
@@ -26,10 +27,7 @@ def add_arguments(self, parser):
         parser.add_argument("--pages", type=str, required=False, default="1")
 
     def handle(self, *args, **options):
-        year = options.get("year")
-        if not year:
-            print("Please specify an audit year")
-            return
+        year = normalize_year_string(options.get("year"))
 
         try:
             pages_str = options["pages"]

backend/census_historical_migration/sac_general_lib/audit_information.py

@@ -7,7 +7,7 @@
 
 from ..base_field_maps import FormFieldMap, FormFieldInDissem
 from ..sac_general_lib.utils import (
-    _create_json_from_db_object,
+    create_json_from_db_object,
 )
 import audit.validators
 from django.conf import settings
@@ -57,10 +57,10 @@
 ]
 
 
-def _get_agency_prefixes(dbkey):
-    """Returns the agency prefixes for the given dbkey."""
+def _get_agency_prefixes(dbkey, year):
+    """Returns the agency prefixes for a given dbkey and audit year."""
     agencies = set()
-    audits = get_audits(dbkey)
+    audits = get_audits(dbkey, year)
 
     for audit_detail in audits:
         agencies.add(string_to_string(audit_detail.CFDA_PREFIX))
@@ -154,8 +154,8 @@ def audit_information(audit_header):
     """Generates audit information JSON."""
 
     results = _get_sp_framework_gaap_results(audit_header)
-    agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY)
-    audit_info = _create_json_from_db_object(audit_header, mappings)
+    agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY, audit_header.AUDITYEAR)
+    audit_info = create_json_from_db_object(audit_header, mappings)
     audit_info = {
         key: results.get(key, audit_info.get(key))
         for key in set(audit_info) | set(results)