Merge pull request #2731 from GSA-TTS/main

backend/audit/intakelib/checks/check_is_a_workbook.py

@@ -1,6 +1,9 @@
 from django.core.exceptions import ValidationError
 import logging
-from audit.intakelib.intermediate_representation import get_sheet_by_name
+from audit.intakelib.intermediate_representation import (
+    get_sheet_by_name,
+    get_range_by_name,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -12,7 +15,8 @@
 # sloppy and still have a coversheet page.
 def is_a_workbook(ir):
     coversheet = get_sheet_by_name(ir, "Coversheet")
-    if not coversheet:
+    version_range = get_range_by_name(ir, "version")
+    if not (coversheet and version_range):
         raise ValidationError(
             (
                 "(O_o)",

backend/audit/intakelib/checks/check_total_amount_expended.py

@@ -10,7 +10,7 @@
 
 # DESCRIPTION
 # The sum of the amount_expended should equal the total_amount_expended
-# B5=SUM(Form!F$2:F$5000)
+# B5=SUM(Form!F$2:F$MAX_ROWS)
 def total_amount_expended_is_correct(ir):
     total_amount_expended_value = get_range_values_by_name(ir, "total_amount_expended")
     amount_expended_values = get_range_values_by_name(ir, "amount_expended")

backend/audit/intakelib/checks/check_version_number.py

@@ -0,0 +1,32 @@
+from django.core.exceptions import ValidationError
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_by_name,
+)
+from audit.intakelib.common import get_message, build_cell_error_tuple
+
+logger = logging.getLogger(__name__)
+
+AUTHORIZED_VERSIONS = {"1.0.0", "1.0.1", "1.0.2", "1.0.3"}
+
+
+# DESCRIPTION
+# This checks if the uploaded workbook version is valid.
+def validate_workbook_version(ir):
+    version_range = get_range_by_name(ir, "version")
+    errors = []
+    for index, version in enumerate(version_range["values"]):
+        # Check if version is not in the set of valid versions
+        if version not in AUTHORIZED_VERSIONS:
+            errors.append(
+                build_cell_error_tuple(
+                    ir,
+                    version_range,
+                    index,
+                    get_message("check_workbook_version").format(version),
+                )
+            )
+
+    if errors:
+        logger.info("Raising a validation error.")
+        raise ValidationError(errors)

backend/audit/intakelib/checks/runners.py

@@ -10,6 +10,7 @@
 from .check_start_and_end_rows_of_all_columns_are_same import (
     start_and_end_rows_of_all_columns_are_same,
 )
+from .check_version_number import validate_workbook_version
 
 ############
 # Federal awards checks
@@ -50,6 +51,7 @@
 
 general_checks = [
     is_a_workbook,
+    validate_workbook_version,
     uei_exists,
     look_for_empty_rows,
     start_and_end_rows_of_all_columns_are_same,

backend/audit/intakelib/common/error_messages.py

@@ -55,5 +55,6 @@
     "check_federal_award_amount_passed_through_not_allowed": "When Federal Award Passed Through is <b>N</b>, Amount Passed Through must be empty",
     "check_loan_balance": "The loan balance is currently set to {}. It should either be a positive number, N/A, or left empty",
     "check_cardinality_of_passthrough_names_and_ids": "You used a <b>|</b> (bar character) to indicate multiple passthrough names and IDs; you must provide equal numbers of names and IDs. You provided <b>{}</b> name{} and <b>{}</b> ID{}",
+    "check_workbook_version": "Single audit workbook template version {} is not supported. Please download the latest workbook and transfer your data to it",
     "check_integer_values": "<b>{}</b> is not a valid integer",
 }

backend/audit/intakelib/intermediate_representation.py

@@ -281,7 +281,7 @@ def extract_workbook_as_ir(file):
         sheet["ranges"] = ranges
         sheets.append(sheet)
 
-    # Remove all the Nones at the bottom of the sheets, since we have 5000 rows of formulas.
+    # Remove all the Nones at the bottom of the sheets, since we have 10000 rows of formulas.
     for sheet in sheets:
         remove_null_rows(sheet)
 

backend/audit/intakelib/transforms/runners.py

@@ -38,6 +38,7 @@
     rename_additional_notes_sheet_to_form_sheet,
 )
 
+from .xform_add_transform_for_cfda_key import generate_cfda_keys
 from .xform_uniform_cluster_names import regenerate_uniform_cluster_names
 from .xform_reformat_prior_references import reformat_prior_references
 
@@ -103,6 +104,7 @@ def run_all_secondary_auditors_transforms(ir):
     convert_number_of_findings_to_integers,
     convert_loan_balance_to_integers_or_na,
     regenerate_uniform_cluster_names,
+    generate_cfda_keys,
 ]
 
 audit_findings_transforms = general_transforms + [

backend/audit/intakelib/transforms/xform_add_transform_for_cfda_key.py

@@ -0,0 +1,21 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+    get_range_by_name,
+    replace_range_by_name,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def generate_cfda_keys(ir):
+    cfda_keys = []
+    federal_agency_prefixes = get_range_by_name(ir, "federal_agency_prefix")
+    three_digit_extensions = get_range_by_name(ir, "three_digit_extension")
+    for prefix, extension in zip(
+        federal_agency_prefixes["values"], three_digit_extensions["values"]
+    ):
+        cfda_keys.append(f"{prefix}.{extension}" if prefix and extension else "")
+
+    xform_ir = replace_range_by_name(ir, "cfda_key", cfda_keys)
+
+    return xform_ir

backend/census_historical_migration/README.md

@@ -0,0 +1,20 @@
+# Census Historical Migration
+
+### How to run the historic data migrator:
+```
+docker compose run web python manage.py historic_data_migrator --email [email protected] \
+  --year 22 \
+  --dbkey 100010
+```
+- The email address currently must be a User in the system. As this has only been run locally so far, it would often be a test account in my local sandbox env.
+- `year` and `dbkey` are optional. The script will use default values for these if they aren't provided.
+
+### How to run the historic workbook generator:
+```
+docker compose run web python manage.py historic_workbook_generator
+  --year 22 \
+  --output <your_output_directory> \
+  --dbkey 100010
+```
+- `year` is optional and defaults to `22`.
+- The `output` directory will be created if it doesn't already exist.

backend/census_historical_migration/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin  # noqa: F401
+
+# Register your models here.

backend/census_historical_migration/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CensusHistoricalMigrationConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "census_historical_migration"

backend/census_historical_migration/management/commands/historic_data_migrator.py

@@ -0,0 +1,61 @@
+import os
+import logging
+import sys
+
+from config.settings import ENVIRONMENT
+from django.core.management.base import BaseCommand
+from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end
+
+CYPRESS_TEST_EMAIL_ADDR = os.getenv("CYPRESS_LOGIN_TEST_EMAIL_AUDITEE")
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("--email", type=str, required=False)
+        parser.add_argument("--dbkeys", type=str, required=False, default="")
+        parser.add_argument("--years", type=str, required=False, default="")
+
+    def handle(self, *args, **options):
+        dbkeys_str = options["dbkeys"]
+        years_str = options["years"]
+        dbkeys = dbkeys_str.split(",")
+        years = years_str.split(",")
+
+        if len(dbkeys) != len(years):
+            logger.error(
+                "Received {} dbkeys and {} years. Must be equal. Exiting.".format(
+                    len(dbkeys), len(years)
+                )
+            )
+            sys.exit(-1)
+
+        lengths = [len(s) == 2 for s in years]
+        if dbkeys_str and years_str and (not all(lengths)):
+            logger.error("Years must be two digits. Exiting.")
+            sys.exit(-2)
+
+        email = options.get("email", CYPRESS_TEST_EMAIL_ADDR)
+
+        defaults = [
+            (182926, 22),
+            (181744, 22),
+            (191734, 22),
+        ]
+
+        if ENVIRONMENT in ["LOCAL", "DEVELOPMENT", "PREVIEW", "STAGING"]:
+            if dbkeys_str and years_str:
+                logger.info(
+                    f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}"
+                )
+                for dbkey, year in zip(dbkeys, years):
+                    run_end_to_end(email, dbkey, year)
+            else:
+                for pair in defaults:
+                    logger.info("Running {}-{} end-to-end".format(pair[0], pair[1]))
+                    run_end_to_end(email, str(pair[0]), str(pair[1]))
+        else:
+            logger.error(
+                "Cannot run end-to-end workbook generation in production. Exiting."
+            )
+            sys.exit(-3)