diff --git a/backend/.profile b/backend/.profile index 2e1e975e5f..8ff630a90d 100644 --- a/backend/.profile +++ b/backend/.profile @@ -1,65 +1,64 @@ #!/bin/bash -set -e -export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +# Source everything; everything is now a function. +# Remember: bash has no idea if a function exists, +# so a typo in a function name will fail silently. Similarly, +# bash has horrible scoping, so use of `local` in functions is +# critical for cleanliness in the startup script. +source tools/util_startup.sh +# This will choose the correct environment +# for local envs (LOCAL or TESTING) and cloud.gov +source tools/setup_env.sh +source tools/migrate_historic_tables.sh +source tools/api_teardown.sh +source tools/migrate_app_tables.sh +source tools/api_standup.sh +source tools/run_collectstatic.sh +source tools/seed_cog_baseline.sh -export https_proxy="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.uri")" -export smtp_proxy_domain="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.domain")" -export smtp_proxy_port="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.port")" -S3_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" -S3_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" -S3_PRIVATE_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" -S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" -export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" +if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then -# Grab the New Relic license key from the newrelic-creds user-provided service instance -export NEW_RELIC_LICENSE_KEY="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "newrelic-creds" ".[][] | select(.name == \$service_name) | .credentials.NEW_RELIC_LICENSE_KEY")" + ##### + # SETUP THE CGOV ENVIRONMENT + setup_env + gonogo "setup_env" -# Set the application name for New Relic telemetry. -export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)-$(echo "$VCAP_APPLICATION" | jq -r .space_name)" + ##### + # MIGRATE HISTORICAL TABLES + # Migrate the historic tables first. + migrate_historic_tables + gonogo "migrate_historic_tables" -# Set the environment name for New Relic telemetry. -export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)" + ##### + # API TEARDOWN + # API has to be deprecated/removed before migration, because + # of tight coupling between schema/views and the dissemination tables + api_teardown + gonogo "api_teardown" -# Set Agent logging to stdout to be captured by CF Logs -export NEW_RELIC_LOG=stdout + ##### + # MIGRATE APP TABLES + migrate_app_tables + gonogo "migrate_app_tables" -# Logging level, (critical, error, warning, info and debug). Default to info -export NEW_RELIC_LOG_LEVEL=info + ##### + # API STANDUP + # Standup the API, which may depend on migration changes + api_standup + gonogo "api_standup" -# https://docs.newrelic.com/docs/security/security-privacy/compliance/fedramp-compliant-endpoints/ -export NEW_RELIC_HOST="gov-collector.newrelic.com" -# https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy -export NEW_RELIC_PROXY_HOST="$https_proxy" + ##### + # COLLECT STATIC + # Do Django things with static files. + run_collectstatic + gonogo "run_collectstatic" -# We only want to run migrate and collecstatic for the first app instance, not -# for additional app instances, so we gate all of this behind CF_INSTANCE_INDEX -# being 0. -if [[ "$CF_INSTANCE_INDEX" == 0 ]]; then - echo 'Starting API schema deprecation' && - python manage.py drop_deprecated_api_schema_and_views && - echo 'Finished API schema deprecation' && - echo 'Dropping API schema' && - python manage.py drop_api_schema && - echo 'Finished dropping API schema' && - echo 'Starting API schema creation' && - python manage.py create_api_schema && - echo 'Finished API schema creation' && - echo 'Starting migrate' && - python manage.py migrate && - python manage.py migrate --database census-to-gsafac-db && - echo 'Finished migrate' && - echo 'Starting API view creation' && - python manage.py create_api_views && - echo 'Finished view creation' && - echo 'Starting collectstatic' && - python manage.py collectstatic --noinput && - echo 'Finished collectstatic' && - echo 'Starting seed_cog_baseline' && - python manage.py seed_cog_baseline && - echo 'Finished seed_cog_baseline' + ##### + # SEED COG/OVER TABLES + # Setup tables for cog/over assignments + seed_cog_baseline + gonogo "seed_cog_baseline" fi # Make psql usable by scripts, for debugging, etc. diff --git a/backend/census_historical_migration/README.md b/backend/census_historical_migration/README.md index c4810cb640..4eb9a4427b 100644 --- a/backend/census_historical_migration/README.md +++ b/backend/census_historical_migration/README.md @@ -68,7 +68,7 @@ To migrate dbkeys for a given year with pagination: docker compose run --rm web python manage.py run_paginated_migration --year 2022 \ --page_size 1000 - --pages 1, 3, 4 + --pages 1,3,4 ``` - `batchSize` and `pages` are optional. The script will use default values for these if they aren't provided. diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx index b1d52714da..1df43aeb3e 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-eins-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx index 1125321434..e80e315304 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/additional-ueis-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx index e6afc6264e..03d7009a66 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/audit-findings-text-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx index 41972d7f54..14ed7510fa 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/corrective-action-plan-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx index 73b58cc070..176a55a261 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-audit-findings-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx index fc518db527..c8466df898 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/federal-awards-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx index ea8c9e20fa..a2110280d0 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/notes-to-sefa-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx index 364b6fceb4..57b38a4f6a 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/177310-22/secondary-auditors-workbook-177310.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx index 8ae37b5a65..6153bf7a47 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-eins-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx index eac899ee39..0f7797140d 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/additional-ueis-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx index 10ac5a73d7..976ad8af66 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/audit-findings-text-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx index dec67962ae..6ca3018bc8 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/corrective-action-plan-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx index aa8923992b..1c4d333cab 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-audit-findings-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx index 1f05e5396c..eb6ddb3f68 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/federal-awards-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx index 08e4c34942..cc411fd8df 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/notes-to-sefa-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx index afdcae7306..bad6b0ee9f 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/180818-22/secondary-auditors-workbook-180818.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx index 1b0d75a8c3..d71fd578b4 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-eins-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx index 86fa772b98..e3e9783280 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/additional-ueis-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/audit-findings-text-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/audit-findings-text-workbook-217653.xlsx index d363e1cbfe..d253fbd433 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/audit-findings-text-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/audit-findings-text-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx index 76658ca329..013541ed85 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/corrective-action-plan-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx index 8e18c99b6d..76eca63821 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-audit-findings-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx index 66b308d942..5ac773a1c6 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/federal-awards-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx index 2c0b4b09e5..ec328e3e88 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/notes-to-sefa-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx index f19e0c9053..7ca9f6aad2 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/217653-22/secondary-auditors-workbook-217653.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx index 0d95ae499c..8b3701c670 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-eins-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx index d8cbd3955f..035cb81273 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/additional-ueis-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx index 9833c60cd0..80f1e82946 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/audit-findings-text-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx index 63e9fb3bef..b5b4e51928 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/corrective-action-plan-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx index 9b48b63e5a..731629b439 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-audit-findings-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx index 626a984056..647072e248 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/federal-awards-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx index b6acba98f7..573cb2b882 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/notes-to-sefa-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx index 607300ea94..8513ec3db4 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/251020-22/secondary-auditors-workbook-251020.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx index 5690995082..9371cb5f5b 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-eins-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx index ebdfc03445..72d2d3e7b3 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/additional-ueis-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx index 2235b936c8..69ec8cd1c4 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/audit-findings-text-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx index c50b8281b1..7425d9ef43 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/corrective-action-plan-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx index eb723170e7..8f9de5920d 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-audit-findings-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx index 71a46021db..6b64e63b72 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/federal-awards-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx index 34b66844c0..c1e02c6b03 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/notes-to-sefa-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx index 44e30d4532..9807df7412 100644 Binary files a/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx and b/backend/census_historical_migration/fixtures/workbooks/should_pass/69688-22/secondary-auditors-workbook-69688.xlsx differ diff --git a/backend/census_historical_migration/historic_data_loader.py b/backend/census_historical_migration/historic_data_loader.py index 7e69406b27..283c2693e6 100644 --- a/backend/census_historical_migration/historic_data_loader.py +++ b/backend/census_historical_migration/historic_data_loader.py @@ -1,4 +1,4 @@ -from .models import ELECAUDITHEADER as Gen +from .models import ELECAUDITHEADER as AuditHeader from .workbooklib.end_to_end_core import run_end_to_end from django.contrib.auth import get_user_model @@ -13,7 +13,7 @@ def load_historic_data_for_year(audit_year, page_size, pages): result_log = {} total_count = error_count = 0 user = create_or_get_user() - submissions_for_year = Gen.objects.filter(AUDITYEAR=audit_year).order_by( + submissions_for_year = AuditHeader.objects.filter(AUDITYEAR=audit_year).order_by( "ELECAUDITHEADERID" ) paginator = Paginator(submissions_for_year, page_size) @@ -27,16 +27,11 @@ def load_historic_data_for_year(audit_year, page_size, pages): ) for submission in page.object_list: - dbkey = submission.DBKEY result = {"success": [], "errors": []} + # Migrate a single submission + run_end_to_end(user, submission, result) - try: - # Migrate a single submission - run_end_to_end(user, dbkey, audit_year, result) - except Exception as exc: - result["errors"].append(f"{exc}") - - result_log[(audit_year, dbkey)] = result + result_log[(audit_year, submission.DBKEY)] = result total_count += 1 if len(result["errors"]) > 0: diff --git a/backend/census_historical_migration/management/commands/historic_data_migrator.py b/backend/census_historical_migration/management/commands/historic_data_migrator.py index 74d0b1ca6f..0f300654b4 100644 --- a/backend/census_historical_migration/management/commands/historic_data_migrator.py +++ b/backend/census_historical_migration/management/commands/historic_data_migrator.py @@ -1,62 +1,65 @@ +from django.core.management.base import BaseCommand import logging import sys +from census_historical_migration.sac_general_lib.utils import ( + normalize_year_string, +) +from census_historical_migration.workbooklib.excel_creation_utils import ( + get_audit_header, +) from census_historical_migration.historic_data_loader import create_or_get_user - -from config.settings import ENVIRONMENT -from django.core.management.base import BaseCommand from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end +from django.conf import settings logger = logging.getLogger(__name__) class Command(BaseCommand): def add_arguments(self, parser): - parser.add_argument("--dbkeys", type=str, required=False, default="") - parser.add_argument("--years", type=str, required=False, default="") + parser.add_argument( + "--dbkeys", type=str, required=False, default="177310,251020" + ) + parser.add_argument("--years", type=str, required=False, default="22,22") - def handle(self, *args, **options): - dbkeys_str = options["dbkeys"] - years_str = options["years"] + def initiate_migration(self, dbkeys_str, years_str): dbkeys = dbkeys_str.split(",") - years = years_str.split(",") - if len(dbkeys) != len(years): - logger.error( - "Received {} dbkeys and {} years. Must be equal. Exiting.".format( - len(dbkeys), len(years) + if years_str: + years = [normalize_year_string(year) for year in years_str.split(",")] + if len(dbkeys) != len(years): + logger.error( + "Received {} dbkeys and {} years. Must be equal. Exiting.".format( + len(dbkeys), len(years) + ) ) - ) - sys.exit(-1) - - lengths = [len(s) == 2 for s in years] - if dbkeys_str and years_str and (not all(lengths)): - logger.error("Years must be two digits. Exiting.") - sys.exit(-2) + sys.exit(-1) user = create_or_get_user() - defaults = [ - (177310, 22), - (251020, 22), - ] + if dbkeys_str and years_str: + logger.info( + f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" + ) + for dbkey, year in zip(dbkeys, years): + logger.info("Running {}-{} end-to-end".format(dbkey, year)) + result = {"success": [], "errors": []} + try: + audit_header = get_audit_header(dbkey, year) + except Exception as e: + logger.error(e) + continue + + run_end_to_end(user, audit_header, result) + logger.info(result) - if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]: - if dbkeys_str and years_str: - logger.info( - f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" - ) - for dbkey, year in zip(dbkeys, years): - result = {"success": [], "errors": []} - run_end_to_end(user, dbkey, year, result) - logger.info(result) - else: - for pair in defaults: - logger.info("Running {}-{} end-to-end".format(pair[0], pair[1])) - result = {"success": [], "errors": []} - run_end_to_end(user, str(pair[0]), str(pair[1]), result) - logger.info(result) + def handle(self, *args, **options): + dbkeys_str = options["dbkeys"] + years_str = options["years"] + + if settings.ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]: + self.initiate_migration(dbkeys_str, years_str) else: logger.error( - "Cannot run end-to-end workbook generation in production. Exiting." + "Cannot run end-to-end historic data migrator in production. Exiting." ) sys.exit(-3) diff --git a/backend/census_historical_migration/management/commands/historic_workbook_generator.py b/backend/census_historical_migration/management/commands/historic_workbook_generator.py index 2f41bb1b55..603cc8ec5c 100644 --- a/backend/census_historical_migration/management/commands/historic_workbook_generator.py +++ b/backend/census_historical_migration/management/commands/historic_workbook_generator.py @@ -1,3 +1,7 @@ +from census_historical_migration.workbooklib.excel_creation_utils import ( + get_audit_header, +) +from census_historical_migration.sac_general_lib.utils import normalize_year_string from census_historical_migration.workbooklib.workbook_builder import ( generate_workbook, ) @@ -44,8 +48,8 @@ def handle(self, *args, **options): # noqa: C901 logger.info(e) logger.info(f"Could not create directory {out_basedir}") sys.exit() - - outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{options["year"]}') + year = normalize_year_string(options["year"]) + outdir = os.path.join(out_basedir, f'{options["dbkey"]}-{year[-2:]}') if not os.path.exists(outdir): try: @@ -56,11 +60,10 @@ def handle(self, *args, **options): # noqa: C901 logger.info("could not create output directory. exiting.") sys.exit() + audit_header = get_audit_header(options["dbkey"], year) json_test_tables = [] for section, fun in sections_to_handlers.items(): - (wb, api_json, _, filename) = generate_workbook( - fun, options["dbkey"], options["year"], section - ) + (wb, api_json, _, filename) = generate_workbook(fun, audit_header, section) if wb: wb_path = os.path.join(outdir, filename) wb.save(wb_path) diff --git a/backend/census_historical_migration/management/commands/run_migration_for_year.py b/backend/census_historical_migration/management/commands/run_migration_for_year.py deleted file mode 100644 index 63f799c061..0000000000 --- a/backend/census_historical_migration/management/commands/run_migration_for_year.py +++ /dev/null @@ -1,28 +0,0 @@ -from ...historic_data_loader import load_historic_data_for_year - -from django.core.management.base import BaseCommand - -import logging - - -logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) - - -class Command(BaseCommand): - help = """ - Migrate from Census tables to GSAFAC tables for a given year - Usage: - manage.py run_migration --year - """ - - def add_arguments(self, parser): - parser.add_argument("--year", help="4-digit Audit Year") - - def handle(self, *args, **options): - year = options.get("year") - if not year: - print("Please specify an audit year") - return - - load_historic_data_for_year(audit_year=year) diff --git a/backend/census_historical_migration/management/commands/run_paginated_migration.py b/backend/census_historical_migration/management/commands/run_paginated_migration.py index eedab80484..24c069ff78 100644 --- a/backend/census_historical_migration/management/commands/run_paginated_migration.py +++ b/backend/census_historical_migration/management/commands/run_paginated_migration.py @@ -1,3 +1,4 @@ +from census_historical_migration.sac_general_lib.utils import normalize_year_string from ...historic_data_loader import load_historic_data_for_year from django.core.management.base import BaseCommand @@ -26,10 +27,7 @@ def add_arguments(self, parser): parser.add_argument("--pages", type=str, required=False, default="1") def handle(self, *args, **options): - year = options.get("year") - if not year: - print("Please specify an audit year") - return + year = normalize_year_string(options.get("year")) try: pages_str = options["pages"] diff --git a/backend/census_historical_migration/sac_general_lib/audit_information.py b/backend/census_historical_migration/sac_general_lib/audit_information.py index bb355fdb45..96a0c418c6 100644 --- a/backend/census_historical_migration/sac_general_lib/audit_information.py +++ b/backend/census_historical_migration/sac_general_lib/audit_information.py @@ -7,7 +7,7 @@ from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( - _create_json_from_db_object, + create_json_from_db_object, ) import audit.validators from django.conf import settings @@ -57,10 +57,10 @@ ] -def _get_agency_prefixes(dbkey): - """Returns the agency prefixes for the given dbkey.""" +def _get_agency_prefixes(dbkey, year): + """Returns the agency prefixes for a given dbkey and audit year.""" agencies = set() - audits = get_audits(dbkey) + audits = get_audits(dbkey, year) for audit_detail in audits: agencies.add(string_to_string(audit_detail.CFDA_PREFIX)) @@ -154,8 +154,8 @@ def audit_information(audit_header): """Generates audit information JSON.""" results = _get_sp_framework_gaap_results(audit_header) - agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY) - audit_info = _create_json_from_db_object(audit_header, mappings) + agencies_prefixes = _get_agency_prefixes(audit_header.DBKEY, audit_header.AUDITYEAR) + audit_info = create_json_from_db_object(audit_header, mappings) audit_info = { key: results.get(key, audit_info.get(key)) for key in set(audit_info) | set(results) diff --git a/backend/census_historical_migration/sac_general_lib/auditee_certification.py b/backend/census_historical_migration/sac_general_lib/auditee_certification.py index cc10ecf0a2..17d3080aa3 100644 --- a/backend/census_historical_migration/sac_general_lib/auditee_certification.py +++ b/backend/census_historical_migration/sac_general_lib/auditee_certification.py @@ -2,7 +2,7 @@ from datetime import date from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( - _create_json_from_db_object, + create_json_from_db_object, ) # The following fields represent checkboxes on the auditee certification form. @@ -41,10 +41,10 @@ def _xform_set_certification_date(auditee_certification): def auditee_certification(audit_header): """Generates auditee certification JSON.""" certification = {} - certification["auditee_certification"] = _create_json_from_db_object( + certification["auditee_certification"] = create_json_from_db_object( audit_header, auditee_certification_mappings ) - certification["auditee_signature"] = _create_json_from_db_object( + certification["auditee_signature"] = create_json_from_db_object( audit_header, auditee_signature_mappings ) certification = _xform_set_certification_date(certification) diff --git a/backend/census_historical_migration/sac_general_lib/auditor_certification.py b/backend/census_historical_migration/sac_general_lib/auditor_certification.py index 89b726a2d5..ba73bb7f3e 100644 --- a/backend/census_historical_migration/sac_general_lib/auditor_certification.py +++ b/backend/census_historical_migration/sac_general_lib/auditor_certification.py @@ -2,7 +2,7 @@ from datetime import date from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( - _create_json_from_db_object, + create_json_from_db_object, ) # The following fields represent checkboxes on the auditor certification form. @@ -38,10 +38,10 @@ def _xform_set_certification_date(auditor_certification): def auditor_certification(audit_header): """Generates auditor certification JSON.""" certification = {} - certification["auditor_certification"] = _create_json_from_db_object( + certification["auditor_certification"] = create_json_from_db_object( audit_header, auditor_certification_mappings ) - certification["auditor_signature"] = _create_json_from_db_object( + certification["auditor_signature"] = create_json_from_db_object( audit_header, auditor_signature_mappings ) certification = _xform_set_certification_date(certification) diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index 8a5f7acbdb..74d9ea9837 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -8,7 +8,7 @@ ) from ..base_field_maps import FormFieldMap, FormFieldInDissem from ..sac_general_lib.utils import ( - _create_json_from_db_object, + create_json_from_db_object, ) import re @@ -169,7 +169,7 @@ def _xform_audit_type(general_information): def general_information(audit_header): """Generates general information JSON.""" - general_information = _create_json_from_db_object(audit_header, mappings) + general_information = create_json_from_db_object(audit_header, mappings) # List of transformation functions transformations = [ diff --git a/backend/census_historical_migration/sac_general_lib/report_id_generator.py b/backend/census_historical_migration/sac_general_lib/report_id_generator.py index eb3d3c9637..4165f00d9c 100644 --- a/backend/census_historical_migration/sac_general_lib/report_id_generator.py +++ b/backend/census_historical_migration/sac_general_lib/report_id_generator.py @@ -3,11 +3,13 @@ ) -def xform_dbkey_to_report_id(audit_header, dbkey): +def xform_dbkey_to_report_id(audit_header): # month = audit_header.fyenddate.split('-')[1] # 2022JUN0001000003 # We start new audits at 1 million. # So, we want 10 digits, and zero-pad for # historic DBKEY report_ids dt = xform_census_date_to_datetime(audit_header.FYENDDATE) - return f"{audit_header.AUDITYEAR}-{dt.month:02}-CENSUS-{dbkey.zfill(10)}" + return ( + f"{audit_header.AUDITYEAR}-{dt.month:02}-CENSUS-{audit_header.DBKEY.zfill(10)}" + ) diff --git a/backend/census_historical_migration/sac_general_lib/sac_creator.py b/backend/census_historical_migration/sac_general_lib/sac_creator.py index 4581829c9d..e70cd98589 100644 --- a/backend/census_historical_migration/sac_general_lib/sac_creator.py +++ b/backend/census_historical_migration/sac_general_lib/sac_creator.py @@ -4,7 +4,6 @@ from django.conf import settings from ..exception_utils import DataMigrationError -from ..workbooklib.excel_creation_utils import get_audit_header from ..sac_general_lib.general_information import ( general_information, ) @@ -24,11 +23,14 @@ logger = logging.getLogger(__name__) -def _create_sac(user, dbkey): +def setup_sac(user, audit_header): """Create a SAC object for the historic data migration.""" + if user is None: + raise DataMigrationError("No user provided to setup sac object") + logger.info(f"Creating a SAC object for {user}") + SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist") - audit_header = get_audit_header(dbkey) - generated_report_id = xform_dbkey_to_report_id(audit_header, dbkey) + generated_report_id = xform_dbkey_to_report_id(audit_header) try: exists = SingleAuditChecklist.objects.get(report_id=generated_report_id) @@ -70,23 +72,5 @@ def _create_sac(user, dbkey): sac.auditor_certification = auditor_certification(audit_header) sac.data_source = settings.CENSUS_DATA_SOURCE sac.save() - logger.info("Created single audit checklist %s", sac) return sac - - -def setup_sac(user, auditee_name, dbkey): - """Create a SAC object for the historic data migration.""" - if user is None: - raise DataMigrationError("No user provided to setup sac object") - logger.info(f"Creating a SAC object for {user}, {auditee_name}") - SingleAuditChecklist = apps.get_model("audit.SingleAuditChecklist") - - sac = SingleAuditChecklist.objects.filter( - submitted_by=user, general_information__auditee_name=auditee_name - ).first() - - logger.info(sac) - if sac is None: - sac = _create_sac(user, dbkey) - return sac diff --git a/backend/census_historical_migration/sac_general_lib/utils.py b/backend/census_historical_migration/sac_general_lib/utils.py index bb83bfc2ca..3b781bcaa0 100644 --- a/backend/census_historical_migration/sac_general_lib/utils.py +++ b/backend/census_historical_migration/sac_general_lib/utils.py @@ -1,13 +1,18 @@ -from datetime import date, datetime -from ..transforms.xform_string_to_date import string_to_date +import logging +from datetime import datetime +import sys + from ..transforms.xform_string_to_string import ( string_to_string, ) from ..transforms.xform_string_to_int import string_to_int from ..transforms.xform_string_to_bool import string_to_bool +logger = logging.getLogger(__name__) + -def _create_json_from_db_object(gobj, mappings): +def create_json_from_db_object(gobj, mappings): + """Constructs a JSON object from a database object using a list of mappings.""" json_obj = {} for mapping in mappings: if mapping.in_db is not None: @@ -25,8 +30,6 @@ def _create_json_from_db_object(gobj, mappings): value = string_to_bool(value) elif mapping.type is int: value = string_to_int(value) - elif mapping.type is date: - value = string_to_date(value) else: value = mapping.type(value) @@ -34,33 +37,28 @@ def _create_json_from_db_object(gobj, mappings): return json_obj -def _census_date_to_datetime(cd): - lookup = { - "JAN": 1, - "FEB": 2, - "MAR": 3, - "APR": 4, - "MAY": 5, - "JUN": 6, - "JUL": 7, - "AUG": 8, - "SEP": 9, - "OCT": 10, - "NOV": 11, - "DEC": 12, - } - parts = cd.split("-") - if len(parts) != 3 or parts[1] not in lookup: - raise ValueError("Invalid date format or month abbreviation in census date") - day, month_abbr, year = parts - month = lookup[month_abbr] - - return date(int(year) + 2000, month, int(day)) - - def xform_census_date_to_datetime(date_string): """Convert a census date string from '%m/%d/%Y %H:%M:%S' format to 'YYYY-MM-DD' format.""" # Parse the string into a datetime object dt = datetime.strptime(date_string, "%m/%d/%Y %H:%M:%S") # Extract and return the date part return dt.date() + + +def normalize_year_string(year_string): + """ + Normalizes a year string to a four-digit year format. + """ + try: + year = int(year_string) + except ValueError: + logger.error("Invalid year string.") + sys.exit(-1) + + if 16 <= year < 23: + return str(year + 2000) + elif 2016 <= year < 2023: + return year_string + else: + logger.error("Invalid year string. Audit year must be between 2016 and 2022") + sys.exit(-1) diff --git a/backend/census_historical_migration/transforms/xform_string_to_date.py b/backend/census_historical_migration/transforms/xform_string_to_date.py deleted file mode 100644 index 1c708e425d..0000000000 --- a/backend/census_historical_migration/transforms/xform_string_to_date.py +++ /dev/null @@ -1,15 +0,0 @@ -from datetime import datetime - - -def string_to_date(value): - """Converts a string to a date.""" - if not isinstance(value, str): - raise ValueError(f"Expected string, got {type(value).__name__}") - - value = value.strip() - - # Check if the string can be converted to a date - try: - return datetime.strptime(value, "%Y-%m-%d").date() - except ValueError: - raise ValueError(f"Cannot convert string to date: '{value}'") diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py index 63ac709da4..53554c6416 100644 --- a/backend/census_historical_migration/workbooklib/additional_eins.py +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -2,7 +2,6 @@ string_to_string, ) from ..workbooklib.excel_creation_utils import ( - get_audit_header, map_simple_columns, generate_dissemination_test_table, set_workbook_uei, @@ -43,27 +42,26 @@ def xform_remove_trailing_decimal_zero(value): ] -def _get_eins(dbkey): - return Eins.objects.filter(DBKEY=dbkey) +def _get_eins(dbkey, year): + return Eins.objects.filter(DBKEY=dbkey, AUDITYEAR=year) -def generate_additional_eins(dbkey, year, outfile): +def generate_additional_eins(audit_header, outfile): """ - Generates additional eins workbook for a given dbkey. + Generates additional eins workbook for a given audit header. """ - logger.info(f"--- generate additional eins {dbkey} {year} ---") + logger.info( + f"--- generate additional eins {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_EINS]) - audit_header = get_audit_header(dbkey) - set_workbook_uei(wb, audit_header.UEI) - - addl_eins = _get_eins(dbkey) + addl_eins = _get_eins(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, addl_eins) wb.save(outfile) - # FIXME - MSHD: The logic below will most likely be removed, see comment in federal_awards.py + table = generate_dissemination_test_table( - audit_header, "additional_eins", dbkey, mappings, addl_eins + audit_header, "additional_eins", mappings, addl_eins ) table["singletons"]["auditee_uei"] = audit_header.UEI return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py index eb1f4c346a..f0180c96f1 100644 --- a/backend/census_historical_migration/workbooklib/additional_ueis.py +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -1,5 +1,4 @@ from ..workbooklib.excel_creation_utils import ( - get_audit_header, map_simple_columns, generate_dissemination_test_table, set_workbook_uei, @@ -24,28 +23,26 @@ ] -def _get_ueis(dbkey): - return Ueis.objects.filter(DBKEY=dbkey) +def _get_ueis(dbkey, year): + return Ueis.objects.filter(DBKEY=dbkey, AUDITYEAR=year) -def generate_additional_ueis(dbkey, year, outfile): +def generate_additional_ueis(audit_header, outfile): """ - Generates additional ueis workbook for a given dbkey. + Generates additional ueis workbook for a given audit header. """ - logger.info(f"--- generate additional ueis {dbkey} {year} ---") + logger.info( + f"--- generate additional ueis {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.ADDITIONAL_UEIS]) - audit_header = get_audit_header(dbkey) set_workbook_uei(wb, audit_header.UEI) - - additional_ueis = _get_ueis(dbkey) + additional_ueis = _get_ueis(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, additional_ueis) wb.save(outfile) - # FIXME - MSHD: The logic below will most likely be removed, see comment in federal_awards.py table = generate_dissemination_test_table( - audit_header, "additional_ueis", dbkey, mappings, additional_ueis + audit_header, "additional_ueis", mappings, additional_ueis ) - table["singletons"]["auditee_uei"] = audit_header.UEI return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py index 263f2cdb44..77b874e1fb 100644 --- a/backend/census_historical_migration/workbooklib/corrective_action_plan.py +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -1,5 +1,4 @@ from ..workbooklib.excel_creation_utils import ( - get_audit_header, map_simple_columns, generate_dissemination_test_table, set_workbook_uei, @@ -30,31 +29,28 @@ ] -def _get_cap_text(dbkey): - return CapText.objects.filter(DBKEY=dbkey).order_by("SEQ_NUMBER") +def _get_cap_text(dbkey, year): + return CapText.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by("SEQ_NUMBER") -def generate_corrective_action_plan(dbkey, year, outfile): +def generate_corrective_action_plan(audit_header, outfile): """ - Generates a corrective action plan workbook for a given dbkey. + Generates a corrective action plan workbook for a given audit header. """ - logger.info(f"--- generate corrective action plan {dbkey} {year} ---") + logger.info( + f"--- generate corrective action plan {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.CORRECTIVE_ACTION_PLAN] ) - audit_header = get_audit_header(dbkey) - set_workbook_uei(wb, audit_header.UEI) - - captexts = _get_cap_text(dbkey) - + captexts = _get_cap_text(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, captexts) wb.save(outfile) table = generate_dissemination_test_table( - audit_header, "corrective_action_plans", dbkey, mappings, captexts + audit_header, "corrective_action_plans", mappings, captexts ) table["singletons"]["auditee_uei"] = audit_header.UEI - return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/end_to_end_core.py b/backend/census_historical_migration/workbooklib/end_to_end_core.py index 1c88ee5488..f9fab2ce5e 100644 --- a/backend/census_historical_migration/workbooklib/end_to_end_core.py +++ b/backend/census_historical_migration/workbooklib/end_to_end_core.py @@ -1,25 +1,14 @@ +from django.conf import settings from ..exception_utils import DataMigrationError -import argparse -import logging -import sys -import math -from config import settings -import os -import jwt -import requests -from datetime import datetime -import traceback - from ..workbooklib.workbook_builder_loader import ( workbook_builder_loader, ) -from ..sac_general_lib.sac_creator import setup_sac from ..workbooklib.workbook_section_handlers import ( sections_to_handlers, ) from ..workbooklib.post_upload_utils import _post_upload_pdf +from ..sac_general_lib.sac_creator import setup_sac from audit.intake_to_dissemination import IntakeToDissemination - from dissemination.models import ( AdditionalEin, AdditionalUei, @@ -33,16 +22,24 @@ SecondaryAuditor, ) +from django.core.exceptions import ValidationError + +import argparse +import logging +import sys +import math +import os +import jwt +import requests +from datetime import datetime +import traceback + + logger = logging.getLogger(__name__) logging.basicConfig() logging.getLogger().setLevel(logging.INFO) parser = argparse.ArgumentParser() -# Peewee runs a really noisy DEBUG log. -pw = logging.getLogger("peewee") -pw.addHandler(logging.StreamHandler()) -pw.setLevel(logging.INFO) - def step_through_certifications(sac): sac.transition_to_ready_for_certification() @@ -53,7 +50,7 @@ def step_through_certifications(sac): sac.save() -def disseminate(sac, year): +def disseminate(sac): logger.info("Invoking movement of data from Intake to Dissemination") for model in [ AdditionalEin, @@ -139,13 +136,13 @@ def _compare_multiline_strings(str1, str2): # Compare line counts if len(lines1) != len(lines2): - print("Line count differs.") + logger.info("Line count differs.") return False # Compare each line for index, (line1, line2) in enumerate(zip(lines1, lines2)): if line1 != line2: - print( + logger.info( f"Difference found on line {index + 1}:\n- {repr(line1)}\n- {repr(line2)}" ) return False @@ -158,10 +155,10 @@ def get_api_values(endpoint, rid, field): res = call_api(api_url, endpoint, rid, field) if res.status_code == 200: - # print(f'{res.status_code} {res.url} {res.json()}') + # logger.info(f'{res.status_code} {res.url} {res.json()}') return list(map(lambda d: d[field], res.json())) else: - print(f"{res.status_code} {res.url}") + logger.error(f"{res.status_code} {res.url}") return [] @@ -181,25 +178,32 @@ def combine_counts(combined, d): def api_check(json_test_tables): combined_summary = {"endpoints": 0, "correct_rows": 0, "incorrect_rows": 0} + for endo in json_test_tables: count(combined_summary, "endpoints") endpoint = endo["endpoint"] report_id = endo["report_id"] - print(f"-------------------- {endpoint} --------------------") summary = {} equality_results = [] + + logger.info(f"-------------------- {endpoint} --------------------") + for row_ndx, row in enumerate(endo["rows"]): count(summary, "total_rows") + if False in equality_results: count(combined_summary, "incorrect_rows") else: count(combined_summary, "correct_rows") + equality_results = [] + for field_ndx, f in enumerate(row["fields"]): # logger.info(f"Checking /{endpoint} {report_id} {f}") # logger.info(f"{get_api_values(endpoint, report_id, f)}") api_values = get_api_values(endpoint, report_id, f) this_api_value = api_values[row_ndx] + # Check if field_ndx exists in row["values"] if field_ndx < len(row["values"]): this_field_value = row["values"][field_ndx] @@ -217,28 +221,29 @@ def api_check(json_test_tables): logger.info( f"Field '{f}' with value '{this_api_value}' at index '{field_ndx}' is missing from test tables 'values'." ) + if all(equality_results): count(summary, "correct_fields") else: count(summary, "incorrect_fields") - sys.exit(-1) + logger.info(summary) combined_summary = combine_counts(combined_summary, summary) + return combined_summary -def run_end_to_end(user, dbkey, year, result): +def run_end_to_end(user, audit_header, result): try: - entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format( - dbkey=dbkey, year=year, date=datetime.now() - ) - sac = setup_sac(user, entity_id, dbkey) + sac = setup_sac(user, audit_header) if sac.general_information["audit_type"] == "alternative-compliance-engagement": - print(f"Skipping ACE audit: {dbkey}") + logger.info( + f"Skipping ACE audit: {audit_header.DBKEY} {audit_header.AUDITYEAR}" + ) raise DataMigrationError("Skipping ACE audit") else: - builder_loader = workbook_builder_loader(user, sac, dbkey, year) + builder_loader = workbook_builder_loader(user, sac, audit_header) json_test_tables = [] for section, fun in sections_to_handlers.items(): @@ -254,14 +259,24 @@ def run_end_to_end(user, dbkey, year, result): result["errors"].append(f"{errors.get('errors')}") return - disseminate(sac, year) + disseminate(sac) combined_summary = api_check(json_test_tables) logger.info(combined_summary) - result["success"].append(f"{sac.report_id} created") except Exception as exc: - tb = traceback.extract_tb(sys.exc_info()[2]) - for frame in tb: - print(f"{frame.filename}:{frame.lineno} {frame.name}: {frame.line}") + error_type = type(exc) + + if error_type == ValidationError: + logger.error(f"ValidationError: {exc}") + elif error_type == DataMigrationError: + logger.error(f"DataMigrationError: {exc.message}") + else: + logger.error(f"Unexpected error type {error_type}: {exc}") + + tb = traceback.extract_tb(sys.exc_info()[2]) + for frame in tb: + logger.error( + f"{frame.filename}:{frame.lineno} {frame.name}: {frame.line}" + ) result["errors"].append(f"{exc}") diff --git a/backend/census_historical_migration/workbooklib/excel_creation_utils.py b/backend/census_historical_migration/workbooklib/excel_creation_utils.py index 866df8dc62..70bca2de32 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation_utils.py +++ b/backend/census_historical_migration/workbooklib/excel_creation_utils.py @@ -131,12 +131,14 @@ def set_workbook_uei(workbook, uei): set_range(workbook, "auditee_uei", [uei]) -def get_audit_header(dbkey): - """Returns the AuditHeader instance for the given dbkey.""" +def get_audit_header(dbkey, year): + """Returns the AuditHeader record for the given dbkey and audit year.""" try: - audit_header = AuditHeader.objects.get(DBKEY=dbkey) + audit_header = AuditHeader.objects.get(DBKEY=dbkey, AUDITYEAR=year) except AuditHeader.DoesNotExist: - raise DataMigrationError(f"No audit header record found for dbkey: {dbkey}") + raise DataMigrationError( + f"No audit header record found for dbkey: {dbkey} and audit year: {year}" + ) return audit_header @@ -176,13 +178,11 @@ def get_template_name_for_section(section): raise ValueError(f"Unknown section {section}") -def generate_dissemination_test_table( - audit_header, api_endpoint, dbkey, mappings, objects -): +def generate_dissemination_test_table(audit_header, api_endpoint, mappings, objects): """Generates a test table for verifying the API queries results.""" table = {"rows": list(), "singletons": dict()} table["endpoint"] = api_endpoint - table["report_id"] = xform_dbkey_to_report_id(audit_header, dbkey) + table["report_id"] = xform_dbkey_to_report_id(audit_header) for o in objects: test_obj = {} @@ -208,6 +208,6 @@ def generate_dissemination_test_table( return table -def get_audits(dbkey): - """Returns the Audits instances for the given dbkey.""" - return Audits.objects.filter(DBKEY=dbkey).order_by("ID") +def get_audits(dbkey, year): + """Returns Audits records for the given dbkey and audit year.""" + return Audits.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by("ID") diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 28c3c7e16e..710d829007 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -2,7 +2,6 @@ string_to_string, ) from ..workbooklib.excel_creation_utils import ( - get_audit_header, get_audits, get_range_values, get_ranges, @@ -17,7 +16,7 @@ ) from ..workbooklib.templates import sections_to_template_paths from audit.fixtures.excel import FORM_SECTIONS -from config import settings +from django.conf import settings from ..models import ( ELECAUDITS as Audits, ELECPASSTHROUGH as Passthrough, @@ -163,7 +162,9 @@ def _get_passthroughs(audits): for index, audit in enumerate(audits): passthroughs = Passthrough.objects.filter( - DBKEY=audit.DBKEY, ELECAUDITSID=audit.ELECAUDITSID + DBKEY=audit.DBKEY, + AUDITYEAR=audit.AUDITYEAR, + ELECAUDITSID=audit.ELECAUDITSID, ).order_by("ID") # This may look like data transformation but it is not exactly the case. # In the audit worksheet, users can enter multiple names (or IDs) separated by a pipe '|' in a single cell. @@ -230,7 +231,7 @@ def _xform_populate_default_loan_balance(loans_at_end, audits): # FIXME - MSHD: _xform_populate_default_award_identification_values is currently unused # as unrequired data transformation will not be part of the first iteration # of the data migration process. -def _xform_populate_default_award_identification_values(audits, dbkey): +def _xform_populate_default_award_identification_values(audits, audit_header): """ Automatically fills in default values for empty additional award identifications. Iterates over a list of audits and their corresponding additional award identifications. @@ -239,13 +240,14 @@ def _xform_populate_default_award_identification_values(audits, dbkey): """ addl_award_identifications = [""] * len(audits) filtered_audits = Audits.objects.filter( - Q(DBKEY=dbkey) & (Q(CFDA__icontains="U") | Q(CFDA__icontains="rd")) + Q(DBKEY=audit_header.DBKEY, AUDITYEAR=audit_header.AUDITYEAR) + & (Q(CFDA__icontains="U") | Q(CFDA__icontains="rd")) ).order_by("ID") for audit in filtered_audits: if audit.AWARDIDENTIFICATION is None or len(audit.AWARDIDENTIFICATION) < 1: addl_award_identifications[ get_list_index(audits, audit.ID) - ] = f"ADDITIONAL AWARD INFO - DBKEY {dbkey}" + ] = f"ADDITIONAL AWARD INFO - DBKEY {audit_header.DBKEY} AUDITYEAR {audit_header.AUDITYEAR}" else: addl_award_identifications[ get_list_index(audits, audit.ID) @@ -253,25 +255,19 @@ def _xform_populate_default_award_identification_values(audits, dbkey): return addl_award_identifications -def generate_federal_awards(dbkey, year, outfile): +def generate_federal_awards(audit_header, outfile): """ - Generates a federal awards workbook for all awards associated with a given dbkey. - - Note: This function assumes that all the audit information in the database - is for the same year. + Generates a federal awards workbook for all awards associated with a given audit header. """ - logger.info(f"--- generate federal awards {dbkey} {year} ---") + logger.info( + f"--- generate federal awards {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED] ) - - audit_header = get_audit_header(dbkey) - set_workbook_uei(wb, audit_header.UEI) - - audits = get_audits(dbkey) - + audits = get_audits(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, audits) (cluster_names, other_cluster_names, state_cluster_names) = _generate_cluster_names( @@ -315,15 +311,10 @@ def generate_federal_awards(dbkey, year, outfile): for audit in audits: total += int(audit.AMOUNT) set_range(wb, "total_amount_expended", [str(total)]) - wb.save(outfile) - # FIXME - MSHD: The test table and the logic around it do not seem necessary to me. - # If there is any chance that the dissemination process allows bogus data to be disseminated, - # we should fix the dissemination process instead by reinforcing the validation logic (intake validation and cross-validation). - # I will create a ticket for the removal of this logic unless someone comes up with a strong reason to keep it. table = generate_dissemination_test_table( - audit_header, "federal_awards", dbkey, mappings, audits + audit_header, "federal_awards", mappings, audits ) award_counter = 1 filtered_mappings = [ diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py index 948c0e767a..2fd4bc59f1 100644 --- a/backend/census_historical_migration/workbooklib/findings.py +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -2,7 +2,6 @@ string_to_string, ) from ..workbooklib.excel_creation_utils import ( - get_audit_header, get_audits, map_simple_columns, generate_dissemination_test_table, @@ -114,37 +113,34 @@ def _get_findings_grid(findings_list): ] -def _get_findings(dbkey): +def _get_findings(dbkey, year): # CFDAs aka ELECAUDITS (or Audits) have elecauditid (FK). Findings have elecauditfindingsid, which is unique. # The linkage here is that a given finding will have an elecauditid. # Multiple findings will have a given elecauditid. That's how to link them. - return Findings.objects.filter(DBKEY=dbkey).order_by("ELECAUDITFINDINGSID") + return Findings.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by( + "ELECAUDITFINDINGSID" + ) -def generate_findings(dbkey, year, outfile): +def generate_findings(audit_header, outfile): """ - Generates a federal awards audit findings workbook for all findings associated with a given dbkey. - - Note: This function assumes that all the audit information in the database - is for the same year. + Generates a federal awards audit findings workbook for all findings associated with a given audit header. """ - logger.info(f"--- generate findings {dbkey} {year} ---") - - audit_header = get_audit_header(dbkey) + logger.info( + f"--- generate findings {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE] ) - set_workbook_uei(wb, audit_header.UEI) - - audits = get_audits(dbkey) + audits = get_audits(audit_header.DBKEY, audit_header.AUDITYEAR) # For each of them, I need to generate an elec -> award mapping. e2a = {} for index, audit in enumerate(audits): e2a[audit.ELECAUDITSID] = f"AWARD-{index+1:04d}" - findings = _get_findings(dbkey) + findings = _get_findings(audit_header.DBKEY, audit_header.AUDITYEAR) award_references = [] for find in findings: @@ -158,9 +154,8 @@ def generate_findings(dbkey, year, outfile): set_range(wb, "is_valid", grid, conversion_fun=str) wb.save(outfile) - # FIXME - MSHD: The logic below will be removed, see comment in federal_award.py. table = generate_dissemination_test_table( - audit_header, "findings", dbkey, mappings, findings + audit_header, "findings", mappings, findings ) for obj, ar in zip(table["rows"], award_references): obj["fields"].append("award_reference") diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py index ed0c9e811d..78a409c1c0 100644 --- a/backend/census_historical_migration/workbooklib/findings_text.py +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -1,5 +1,4 @@ from ..workbooklib.excel_creation_utils import ( - get_audit_header, map_simple_columns, generate_dissemination_test_table, set_workbook_uei, @@ -29,30 +28,30 @@ ] -def _get_findings_texts(dbkey): - return FindingsText.objects.filter(DBKEY=dbkey).order_by("SEQ_NUMBER") +def _get_findings_texts(dbkey, year): + return FindingsText.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by( + "SEQ_NUMBER" + ) -def generate_findings_text(dbkey, year, outfile): +def generate_findings_text(audit_header, outfile): """ - Generates a findings text workbook for a given dbkey. - - Note: This function assumes that all the findings text - information in the database is related to the same year. + Generates a findings text workbook for a given audit header. """ - logger.info(f"--- generate findings text {dbkey} {year} ---") + logger.info( + f"--- generate findings text {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.FINDINGS_TEXT]) - audit_header = get_audit_header(dbkey) set_workbook_uei(wb, audit_header.UEI) - findings_texts = _get_findings_texts(dbkey) + findings_texts = _get_findings_texts(audit_header.DBKEY, audit_header.AUDITYEAR) map_simple_columns(wb, mappings, findings_texts) wb.save(outfile) table = generate_dissemination_test_table( - audit_header, "findings_text", dbkey, mappings, findings_texts + audit_header, "findings_text", mappings, findings_texts ) table["singletons"]["auditee_uei"] = audit_header.UEI diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index 11aba8aa8d..2553ab00ca 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -2,7 +2,6 @@ from ..transforms.xform_string_to_string import string_to_string from ..models import ELECNOTES as Notes from ..workbooklib.excel_creation_utils import ( - get_audit_header, set_range, map_simple_columns, generate_dissemination_test_table, @@ -76,13 +75,13 @@ def xform_is_minimis_rate_used(rate_content): raise DataMigrationError("Unable to determine if the de minimis rate was used.") -def _get_accounting_policies(dbkey): +def _get_accounting_policies(dbkey, year): # https://facdissem.census.gov/Documents/DataDownloadKey.xlsx # The TYPEID column determines which field in the form a given row corresponds to. # TYPEID=1 is the description of significant accounting policies. - """Get the accounting policies for a given dbkey.""" + """Get the accounting policies for a given dbkey and audit year.""" try: - note = Notes.objects.get(DBKEY=dbkey, TYPE_ID="1") + note = Notes.objects.get(DBKEY=dbkey, AUDITYEAR=year, TYPE_ID="1") content = string_to_string(note.CONTENT) except Notes.DoesNotExist: logger.info(f"No accounting policies found for dbkey: {dbkey}") @@ -90,13 +89,13 @@ def _get_accounting_policies(dbkey): return content -def _get_minimis_cost_rate(dbkey): - """Get the De Minimis cost rate for a given dbkey.""" +def _get_minimis_cost_rate(dbkey, year): + """Get the De Minimis cost rate for a given dbkey and audit year.""" # https://facdissem.census.gov/Documents/DataDownloadKey.xlsx # The TYPEID column determines which field in the form a given row corresponds to. # TYPEID=2 is the De Minimis cost rate. try: - note = Notes.objects.get(DBKEY=dbkey, TYPE_ID="2") + note = Notes.objects.get(DBKEY=dbkey, AUDITYEAR=year, TYPE_ID="2") rate = string_to_string(note.CONTENT) except Notes.DoesNotExist: logger.info(f"De Minimis cost rate not found for dbkey: {dbkey}") @@ -104,28 +103,32 @@ def _get_minimis_cost_rate(dbkey): return rate -def _get_notes(dbkey): - """Get the notes for a given dbkey.""" +def _get_notes(dbkey, year): + """Get the notes for a given dbkey and audit year.""" # https://facdissem.census.gov/Documents/DataDownloadKey.xlsx # The TYPEID column determines which field in the form a given row corresponds to. # TYPEID=3 is for notes, which have sequence numbers... that must align somewhere. - return Notes.objects.filter(DBKEY=dbkey, TYPE_ID="3").order_by("SEQ_NUMBER") + return Notes.objects.filter(DBKEY=dbkey, AUDITYEAR=year, TYPE_ID="3").order_by( + "SEQ_NUMBER" + ) -def generate_notes_to_sefa(dbkey, year, outfile): +def generate_notes_to_sefa(audit_header, outfile): """ - Generates notes to SEFA workbook for a given dbkey. + Generates notes to SEFA workbook for a given audit header. """ - logger.info(f"--- generate notes to sefa {dbkey} {year}---") + logger.info( + f"--- generate notes to sefa {audit_header.DBKEY} {audit_header.AUDITYEAR}---" + ) wb = pyxl.load_workbook(sections_to_template_paths[FORM_SECTIONS.NOTES_TO_SEFA]) - audit_header = get_audit_header(dbkey) set_workbook_uei(wb, audit_header.UEI) - - notes = _get_notes(dbkey) - rate_content = _get_minimis_cost_rate(dbkey) - policies_content = _get_accounting_policies(dbkey) + notes = _get_notes(audit_header.DBKEY, audit_header.AUDITYEAR) + rate_content = _get_minimis_cost_rate(audit_header.DBKEY, audit_header.AUDITYEAR) + policies_content = _get_accounting_policies( + audit_header.DBKEY, audit_header.AUDITYEAR + ) is_minimis_rate_used = xform_is_minimis_rate_used(rate_content) set_range(wb, "accounting_policies", [policies_content]) @@ -144,9 +147,8 @@ def generate_notes_to_sefa(dbkey, year, outfile): wb.save(outfile) table = generate_dissemination_test_table( - audit_header, "notes_to_sefa", dbkey, mappings, notes + audit_header, "notes_to_sefa", mappings, notes ) - table["singletons"]["accounting_policies"] = policies_content table["singletons"]["is_minimis_rate_used"] = is_minimis_rate_used table["singletons"]["rate_explained"] = rate_content diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index 760a34f255..a98a03f49d 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -1,6 +1,5 @@ from ..transforms.xform_string_to_string import string_to_string from ..workbooklib.excel_creation_utils import ( - get_audit_header, map_simple_columns, generate_dissemination_test_table, set_workbook_uei, @@ -75,30 +74,30 @@ def xform_add_hyphen_to_zip(zip): ] -def _get_secondary_auditors(dbkey): - return Caps.objects.filter(DBKEY=dbkey) +def _get_secondary_auditors(dbkey, year): + return Caps.objects.filter(DBKEY=dbkey, AUDITYEAR=year) -def generate_secondary_auditors(dbkey, year, outfile): +def generate_secondary_auditors(audit_header, outfile): """ - Generates secondary auditor workbook for a given dbkey. + Generates secondary auditor workbook for a given audit header. """ - logger.info(f"--- generate secondary auditors {dbkey} {year} ---") + logger.info( + f"--- generate secondary auditors {audit_header.DBKEY} {audit_header.AUDITYEAR} ---" + ) wb = pyxl.load_workbook( sections_to_template_paths[FORM_SECTIONS.SECONDARY_AUDITORS] ) - audit_header = get_audit_header(dbkey) set_workbook_uei(wb, audit_header.UEI) - - secondary_auditors = _get_secondary_auditors(dbkey) + secondary_auditors = _get_secondary_auditors( + audit_header.DBKEY, audit_header.AUDITYEAR + ) map_simple_columns(wb, mappings, secondary_auditors) - wb.save(outfile) - # FIXME - MSHD: The logic below will most likely be removed, see comment in federal_awards.py table = generate_dissemination_test_table( - audit_header, "secondary_auditors", dbkey, mappings, secondary_auditors + audit_header, "secondary_auditors", mappings, secondary_auditors ) table["singletons"]["auditee_uei"] = audit_header.UEI diff --git a/backend/census_historical_migration/workbooklib/workbook_builder.py b/backend/census_historical_migration/workbooklib/workbook_builder.py index 9d2bcc9b45..51572dbfef 100644 --- a/backend/census_historical_migration/workbooklib/workbook_builder.py +++ b/backend/census_historical_migration/workbooklib/workbook_builder.py @@ -17,10 +17,10 @@ def _make_excel_file(filename, f_obj): return file -def generate_workbook(workbook_generator, dbkey, year, section): +def generate_workbook(workbook_generator, audit_header, section): """ - Generates a workbook in memory using the workbook_generator for a specific - 'dbkey', 'year', and 'section'. Returns the workbook object, its JSON data representation, + Generates a workbook in memory using the workbook_generator for a given audit_header + and section template name. Returns the workbook object, its JSON data representation, the Excel file as a SimpleUploadedFile object, and the filename. """ with MemoryFS() as mem_fs: @@ -28,11 +28,11 @@ def generate_workbook(workbook_generator, dbkey, year, section): filename = ( get_template_name_for_section(section) .replace(".xlsx", "-{}.xlsx") - .format(dbkey) + .format(audit_header.DBKEY) ) with mem_fs.openbin(filename, mode="w") as outfile: # Generate the workbook object along with the API JSON representation - wb, json_data = workbook_generator(dbkey, year, outfile) + wb, json_data = workbook_generator(audit_header, outfile) # Re-open the file in read mode to create an Excel file object with mem_fs.openbin(filename, mode="r") as outfile: diff --git a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py index 07c2bd3a94..8ab5b021cc 100644 --- a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py +++ b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -def workbook_builder_loader(user, sac, dbkey, year): +def workbook_builder_loader(user, sac, audit_header): """ Returns a nested function '_loader' that, when called with a workbook generator and a section, generates a workbook for the section, uploads it to SAC, @@ -17,7 +17,7 @@ def workbook_builder_loader(user, sac, dbkey, year): def _loader(workbook_generator, section): wb, json_data, excel_file, filename = generate_workbook( - workbook_generator, dbkey, year, section + workbook_generator, audit_header, section ) if user: diff --git a/backend/run.sh b/backend/run.sh index 1d428ae0f7..9d52d5ab6d 100755 --- a/backend/run.sh +++ b/backend/run.sh @@ -1,42 +1,56 @@ #!/bin/bash -if [[ -n "${ENV}" ]]; then - echo "Environment set as: ${ENV}" -else - echo "No environment variable ${ENV} is set!" -fi; - -sleep 10 - -if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then - export AWS_PRIVATE_ACCESS_KEY_ID=longtest - export AWS_PRIVATE_SECRET_ACCESS_KEY=longtest - export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000" - mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin - mc mb myminio/gsa-fac-private-s3 - mc mb myminio/fac-census-to-gsafac-s3 - mc admin user svcacct add --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" myminio minioadmin -fi; - -# Migrate first -python manage.py migrate -python manage.py migrate --database census-to-gsafac-db - - -echo 'Starting API schema deprecation' && -python manage.py drop_deprecated_api_schema_and_views && -echo 'Finished API schema deprecation' && -echo 'Dropping API schema' && -python manage.py drop_api_schema && -echo 'Finished dropping API schema' && -echo 'Starting API schema creation' && -python manage.py create_api_schema && -echo 'Finished API schema creation' && -echo 'Starting API view creation' && -python manage.py create_api_views && -echo 'Finished view creation' && -echo 'Starting seed_cog_baseline' && -python manage.py seed_cog_baseline && -echo 'Finished seed_cog_baseline' +# Source everything; everything is now a function. +# Remember: bash has no idea if a function exists, +# so a typo in a function name will fail silently. Similarly, +# bash has horrible scoping, so use of `local` in functions is +# critical for cleanliness in the startup script. +source tools/util_startup.sh +# This will choose the correct environment +# for local envs (LOCAL or TESTING) and cloud.gov +source tools/setup_env.sh +source tools/migrate_historic_tables.sh +source tools/api_teardown.sh +source tools/migrate_app_tables.sh +source tools/api_standup.sh +source tools/seed_cog_baseline.sh +##### +# SETUP THE LOCAL ENVIRONMENT +setup_env +gonogo "setup_env" + +##### +# MIGRATE HISTORICAL TABLES +# Migrate the historic tables first. +migrate_historic_tables +gonogo "migrate_historic_tables" + +##### +# API TEARDOWN +# API has to be deprecated/removed before migration, because +# of tight coupling between schema/views and the dissemination tables +api_teardown +gonogo "api_teardown" + +##### +# MIGRATE APP TABLES +migrate_app_tables +gonogo "migrate_app_tables" + +##### +# API STANDUP +# Standup the API, which may depend on migration changes +api_standup +gonogo "api_standup" + +##### +# SEED COG/OVER TABLES +# Setup tables for cog/over assignments +seed_cog_baseline +gonogo "seed_cog_baseline" + +##### +# LAUNCH THE APP +# We will have died long ago if things didn't work. npm run dev & python manage.py runserver 0.0.0.0:8000 diff --git a/backend/tools/api_standup.sh b/backend/tools/api_standup.sh new file mode 100644 index 0000000000..57d6079520 --- /dev/null +++ b/backend/tools/api_standup.sh @@ -0,0 +1,29 @@ +source tools/util_startup.sh + +function api_standup { + startup_log "API_STANDUP" "BEGIN" + + # First create non-managed tables + startup_log "CREATE_API_ACCESS_TABLES" "BEGIN" + python manage.py create_api_access_tables + local d1=$? + startup_log "CREATE_API_ACCESS_TABLES" "END" + + # Bring the API back, possibly installing a new API + startup_log "CREATE_API_SCHEMA" "BEGIN" + python manage.py create_api_schema + local d2=$? + startup_log "CREATE_API_SCHEMA" "END" + + startup_log "CREATE_API_VIEWS" "BEGIN" + python manage.py create_api_views && + local d3=$? + startup_log "CREATE_API_VIEWS" "END" + + startup_log "API_STANDUP" "END" + + result=$(($d1 + $d2 + $d3)) + # If these are all zero, we're all good. + return $result +} + diff --git a/backend/tools/api_teardown.sh b/backend/tools/api_teardown.sh new file mode 100644 index 0000000000..80292d3181 --- /dev/null +++ b/backend/tools/api_teardown.sh @@ -0,0 +1,20 @@ +source tools/util_startup.sh + +function api_teardown { + startup_log "API_TEARDOWN" "BEGIN" + + startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "BEGIN" + python manage.py drop_deprecated_api_schema_and_views + local d1=$? + startup_log "DROP_DEPRECATED_API_SCHEMA_AND_VIEWS" "END" + startup_log "DROP_API_SCHEMA" "BEGIN" + python manage.py drop_api_schema + local d2=$? + startup_log "DROP_API_SCHEMA" "END" + + startup_log "API_TEARDOWN" "END" + + result=$(($d1 + $d2)) + # If these are both zero, we're all good. + return $result +} diff --git a/backend/tools/migrate_app_tables.sh b/backend/tools/migrate_app_tables.sh new file mode 100644 index 0000000000..aef200d092 --- /dev/null +++ b/backend/tools/migrate_app_tables.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function migrate_app_tables { + startup_log "MIGRATE_APP_TABLES" "BEGIN" + python manage.py migrate + local result=$? + startup_log "MIGRATE_APP_TABLES" "END" + return $result +} diff --git a/backend/tools/migrate_historic_tables.sh b/backend/tools/migrate_historic_tables.sh new file mode 100644 index 0000000000..6454642bad --- /dev/null +++ b/backend/tools/migrate_historic_tables.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function migrate_historic_tables { + startup_log "HISTORIC_TABLE_MIGRATION" "BEGIN" + python manage.py migrate --database census-to-gsafac-db + local result=$? + startup_log "HISTORIC_TABLE_MIGRATION" "END" + return $result +} diff --git a/backend/tools/run_collectstatic.sh b/backend/tools/run_collectstatic.sh new file mode 100644 index 0000000000..18f09549a4 --- /dev/null +++ b/backend/tools/run_collectstatic.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function run_collectstatic { + startup_log "RUN_COLLECTSTATIC" "BEGIN" + python manage.py collectstatic --noinput && + local result=$? + startup_log "RUN_COLLECTSTATIC" "END" + return $result +} diff --git a/backend/tools/seed_cog_baseline.sh b/backend/tools/seed_cog_baseline.sh new file mode 100644 index 0000000000..3dc5eb4f66 --- /dev/null +++ b/backend/tools/seed_cog_baseline.sh @@ -0,0 +1,9 @@ +source tools/util_startup.sh + +function seed_cog_baseline { + startup_log "SEED_COG_BASELINE" "BEGIN" + python manage.py seed_cog_baseline + local result=$? + startup_log "SEED_COG_BASELINE" "END" + return $result +} diff --git a/backend/tools/setup_cgov_env.sh b/backend/tools/setup_cgov_env.sh new file mode 100644 index 0000000000..73f6c5c572 --- /dev/null +++ b/backend/tools/setup_cgov_env.sh @@ -0,0 +1,39 @@ +source tools/util_startup.sh + +function setup_cgov_env { + set -e + + export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt + export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt + + export https_proxy="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.uri")" + export smtp_proxy_domain="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.domain")" + export smtp_proxy_port="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "smtp-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.port")" + + S3_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" + S3_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-public-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" + S3_PRIVATE_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.endpoint")" + S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY="$(echo $VCAP_SERVICES | jq --raw-output --arg service_name "fac-private-s3" ".[][] | select(.name == \$service_name) | .credentials.fips_endpoint")" + export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_ENDPOINT_FOR_NO_PROXY},${S3_PRIVATE_FIPS_ENDPOINT_FOR_NO_PROXY},apps.internal" + + # Grab the New Relic license key from the newrelic-creds user-provided service instance + export NEW_RELIC_LICENSE_KEY="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "newrelic-creds" ".[][] | select(.name == \$service_name) | .credentials.NEW_RELIC_LICENSE_KEY")" + + # Set the application name for New Relic telemetry. + export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)-$(echo "$VCAP_APPLICATION" | jq -r .space_name)" + + # Set the environment name for New Relic telemetry. + export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)" + + # Set Agent logging to stdout to be captured by CF Logs + export NEW_RELIC_LOG=stdout + + # Logging level, (critical, error, warning, info and debug). Default to info + export NEW_RELIC_LOG_LEVEL=info + + # https://docs.newrelic.com/docs/security/security-privacy/compliance/fedramp-compliant-endpoints/ + export NEW_RELIC_HOST="gov-collector.newrelic.com" + # https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy + export NEW_RELIC_PROXY_HOST="$https_proxy" + return 0 +} diff --git a/backend/tools/setup_env.sh b/backend/tools/setup_env.sh new file mode 100644 index 0000000000..5839f92de0 --- /dev/null +++ b/backend/tools/setup_env.sh @@ -0,0 +1,21 @@ +source tools/setup_local_env.sh +source tools/setup_cgov_env.sh + +function setup_env { + if [[ -n "${ENV}" ]]; then + startup_log "LOCAL_ENV" "Environment set as: ${ENV}" + else + startup_log "LOCAL_ENV" "No environment variable ${ENV} is set!" + return -1 + fi; + + local result=0 + if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then + setup_local_env + result=$? + else + setup_cgov_env + result=$? + fi; + return $result +} diff --git a/backend/tools/setup_local_env.sh b/backend/tools/setup_local_env.sh new file mode 100644 index 0000000000..62b01affb0 --- /dev/null +++ b/backend/tools/setup_local_env.sh @@ -0,0 +1,18 @@ +source tools/util_startup.sh + +function setup_local_env { + + if [[ "${ENV}" == "LOCAL" || "${ENV}" == "TESTING" ]]; then + startup_log "LOCAL_ENV" "We are in a local envirnoment." + export AWS_PRIVATE_ACCESS_KEY_ID=longtest + export AWS_PRIVATE_SECRET_ACCESS_KEY=longtest + export AWS_S3_PRIVATE_ENDPOINT="http://minio:9000" + mc alias set myminio "${AWS_S3_PRIVATE_ENDPOINT}" minioadmin minioadmin + # Do nothing if the bucket already exists. + # https://min.io/docs/minio/linux/reference/minio-mc/mc-mb.html + mc mb --ignore-existing myminio/gsa-fac-private-s3 + mc mb --ignore-existing myminio/fac-census-to-gsafac-s3 + mc admin user svcacct add --access-key="${AWS_PRIVATE_ACCESS_KEY_ID}" --secret-key="${AWS_PRIVATE_SECRET_ACCESS_KEY}" myminio minioadmin + return 0 + fi; +} diff --git a/backend/tools/util_startup.sh b/backend/tools/util_startup.sh new file mode 100644 index 0000000000..ca54205ab9 --- /dev/null +++ b/backend/tools/util_startup.sh @@ -0,0 +1,16 @@ +function startup_log { + local tag="$1" + local msg="$2" + echo STARTUP $tag $msg +} + +# gonogo +# helps determine if we should continue or quit +function gonogo { + if [ $? -eq 0 ]; then + startup_log "STARTUP_CHECK" "$1 PASS" + else + startup_log "STARTUP_CHECK" "$1 FAIL" + exit -1 + fi +}