From f3eb037263f6673923079d21d795cb5830c5aa84 Mon Sep 17 00:00:00 2001 From: "Hassan D. M. Sambo" Date: Tue, 26 Dec 2023 12:52:30 -0500 Subject: [PATCH 1/2] 3066 add test table to general information and improve existing test tables (#3078) * #3066 Added logic to construct general information api test table, restructured end_to_end_core.py * #3066 Added logic to construct general information api test table, restructured end_to_end_core.py * #3066 Updated existing api test table to remove redundant uei field. This field is now checked as part of general information * #3066 Updated queries to filter out empty ueis and eins * #3066 Refactoring + cleaning * # Bug fix * #Consolidate default int * #Updated migrator action * #3066 bug fix * Linting --- .github/workflows/historic-data-migrator.yml | 2 +- .../api_test_helpers.py | 62 +++++++ .../{workbooklib => }/end_to_end_core.py | 162 ++++++++++-------- .../historic_data_loader.py | 6 +- .../management/commands/csv_to_postgres.py | 12 +- .../commands/historic_data_migrator.py | 2 +- .../sac_general_lib/audit_information.py | 9 +- .../sac_general_lib/general_information.py | 8 +- .../sac_general_lib/sac_creator.py | 13 +- .../test_federal_awards_xforms.py | 6 +- .../workbooklib/additional_eins.py | 6 +- .../workbooklib/additional_ueis.py | 6 +- .../workbooklib/corrective_action_plan.py | 4 +- .../workbooklib/excel_creation_utils.py | 34 ---- .../workbooklib/federal_awards.py | 141 ++++++++------- .../workbooklib/findings.py | 9 +- .../workbooklib/findings_text.py | 3 +- .../workbooklib/notes_to_sefa.py | 11 +- .../workbooklib/post_upload_utils.py | 2 +- .../workbooklib/secondary_auditors.py | 3 +- .../workbooklib/workbook_builder_loader.py | 4 +- backend/config/settings.py | 1 + 22 files changed, 281 insertions(+), 225 deletions(-) create mode 100644 backend/census_historical_migration/api_test_helpers.py rename backend/census_historical_migration/{workbooklib => }/end_to_end_core.py (66%) diff --git a/.github/workflows/historic-data-migrator.yml b/.github/workflows/historic-data-migrator.yml index 0b17a42c1b..105eb2da97 100644 --- a/.github/workflows/historic-data-migrator.yml +++ b/.github/workflows/historic-data-migrator.yml @@ -43,4 +43,4 @@ jobs: cf_password: ${{ secrets.CF_PASSWORD }} cf_org: gsa-tts-oros-fac cf_space: ${{ env.space }} - command: cf run-task gsa-fac -k 2G -m 2G --name historic_data_migrator --command "python manage.py historic_data_migrator --dbkeys ${{ inputs.dbkeys }} --years ${{ inputs.years }} --email ${{ inputs.email }}" + command: cf run-task gsa-fac -k 2G -m 2G --name historic_data_migrator --command "python manage.py historic_data_migrator --dbkeys ${{ inputs.dbkeys }} --years ${{ inputs.years }}" diff --git a/backend/census_historical_migration/api_test_helpers.py b/backend/census_historical_migration/api_test_helpers.py new file mode 100644 index 0000000000..6cba0ce84f --- /dev/null +++ b/backend/census_historical_migration/api_test_helpers.py @@ -0,0 +1,62 @@ +from audit.utils import Util +from .base_field_maps import FormFieldInDissem, WorkbookFieldInDissem +from .sac_general_lib.report_id_generator import xform_dbkey_to_report_id +from .workbooklib.excel_creation_utils import apply_conversion_function + + +def generate_dissemination_test_table( + audit_header, api_endpoint, mappings=None, objects=None +): + """Generates a test table for verifying the API queries results.""" + table = {} + table["endpoint"] = api_endpoint + table["report_id"] = xform_dbkey_to_report_id(audit_header) + + if mappings and objects: + table["rows"] = list() + + for o in objects: + test_obj = {} + test_obj["fields"] = [] + test_obj["values"] = [] + for m in mappings: + # What if we only test non-null values? + raw_value = getattr(o, m.in_db, None) + attribute_value = apply_conversion_function( + raw_value, m.default, m.type + ) + if (attribute_value is not None) and (attribute_value != ""): + if m.in_dissem == WorkbookFieldInDissem: + test_obj["fields"].append(m.in_sheet) + test_obj["values"].append(m.type(attribute_value)) + else: + test_obj["fields"].append(m.in_dissem) + test_obj["values"].append(m.type(attribute_value)) + + table["rows"].append(test_obj) + else: + table["singletons"] = dict() + + return table + + +def extract_api_data(mappings, section_object): + """Extract data for verifying the API queries results.""" + table = {} + + for item in mappings: + value = section_object[item.in_form] + + # Apply same transformations as in `intake_to_dissemination.py` + if item.type == bool: + value = Util.bool_to_yes_no(value) + elif item.type == list: + value = Util.json_array_to_str(value) + + if item.in_dissem: + if item.in_dissem == FormFieldInDissem: + table[item.in_form] = value + else: + table[item.in_dissem] = value + + return table diff --git a/backend/census_historical_migration/workbooklib/end_to_end_core.py b/backend/census_historical_migration/end_to_end_core.py similarity index 66% rename from backend/census_historical_migration/workbooklib/end_to_end_core.py rename to backend/census_historical_migration/end_to_end_core.py index 4e322340fc..b9370159d0 100644 --- a/backend/census_historical_migration/workbooklib/end_to_end_core.py +++ b/backend/census_historical_migration/end_to_end_core.py @@ -1,16 +1,16 @@ -from .post_upload_utils import record_dummy_pdf_object -from ..exception_utils import ( +from .workbooklib.post_upload_utils import record_dummy_pdf_object +from .exception_utils import ( DataMigrationError, DataMigrationValueError, ) -from ..workbooklib.workbook_builder_loader import ( +from .workbooklib.workbook_builder_loader import ( workbook_builder_loader, ) -from ..workbooklib.workbook_section_handlers import ( +from .workbooklib.workbook_section_handlers import ( sections_to_handlers, ) -from ..sac_general_lib.sac_creator import setup_sac -from ..models import ( +from .sac_general_lib.sac_creator import setup_sac +from .models import ( ReportMigrationStatus, MigrationErrorDetail, ) @@ -51,18 +51,16 @@ def step_through_certifications(sac): - stati = [ - SingleAuditChecklist.STATUS.IN_PROGRESS, - SingleAuditChecklist.STATUS.READY_FOR_CERTIFICATION, - SingleAuditChecklist.STATUS.AUDITOR_CERTIFIED, - SingleAuditChecklist.STATUS.AUDITEE_CERTIFIED, - SingleAuditChecklist.STATUS.CERTIFIED, - SingleAuditChecklist.STATUS.SUBMITTED, - SingleAuditChecklist.STATUS.DISSEMINATED, - ] - for status in stati: - sac.transition_name.append(status) - sac.transition_date.append(datetime.now(timezone.utc)) + sac.transition_to_ready_for_certification() + sac.transition_to_auditor_certified() + sac.transition_to_auditee_certified() + + # FIXME-MSHD: We have no method transition_to_certified() + sac.transition_name.append(SingleAuditChecklist.STATUS.CERTIFIED) + sac.transition_date.append(datetime.now(timezone.utc)) + + sac.transition_to_submitted() + sac.transition_to_disseminated() sac.save() @@ -128,20 +126,20 @@ def are_they_both_none_or_empty(a, b): return a_val and b_val -def check_equality(in_wb, in_json): +def check_equality(in_wb, in_api): # Type requirement is sometimes just 'N' - if in_wb in ["Y", "N"] and isinstance(in_json, bool): - return (True if in_wb == "Y" else False) == in_json - elif just_numbers(in_wb) and just_numbers(in_json): + if in_wb in ["Y", "N"] and isinstance(in_api, bool): + return (True if in_wb == "Y" else False) == in_api + elif just_numbers(in_wb) and just_numbers(in_api): return ( - True if math.isclose(float(in_wb), float(in_json), rel_tol=1e-1) else False + True if math.isclose(float(in_wb), float(in_api), rel_tol=1e-1) else False ) - elif isinstance(in_wb, str) and isinstance(in_json, str): - return _compare_multiline_strings(in_wb, in_json) - elif in_wb is None or in_json is None: - return are_they_both_none_or_empty(in_wb, in_json) + elif isinstance(in_wb, str) and isinstance(in_api, str): + return _compare_multiline_strings(in_wb, in_api) + elif in_wb is None or in_api is None: + return are_they_both_none_or_empty(in_wb, in_api) else: - return in_wb == in_json + return str(in_wb) == str(in_api) def _compare_multiline_strings(str1, str2): @@ -192,56 +190,73 @@ def combine_counts(combined, d): return combined +def process_singletons(endo, summary): + """Process the singletons in the JSON test table""" + for field, value in endo.get("singletons", {}).items(): + api_values = get_api_values(endo["endpoint"], endo["report_id"], field) + eq = check_equality(value, api_values[0]) + if eq: + count(summary, "correct_fields") + else: + logger.info( + f"Does not match. [eq {eq}] [field {field}] [field val {value}] != [api val {api_values[0]}]" + ) + count(summary, "incorrect_fields") + + +def process_rows(endo, combined_summary, summary): + """Process the rows in the JSON test table""" + rows = endo.get("rows", []) + equality_results = [] + for row_ndx, row in enumerate(rows): + count(summary, "total_rows") + + if False in equality_results: + count(combined_summary, "incorrect_rows") + else: + count(combined_summary, "correct_rows") + + equality_results = [] + + for field_ndx, f in enumerate(row["fields"]): + # logger.info(f"Checking /{endo["endpoint"]} {endo["report_id"]} {f}") + # logger.info(f"{get_api_values(endo["endpoint"], endo["report_id"], f)}") + api_values = get_api_values(endo["endpoint"], endo["report_id"], f) + this_api_value = api_values[row_ndx] + if field_ndx < len(row["values"]): + this_field_value = row["values"][field_ndx] + eq = check_equality(this_field_value, this_api_value) + if not eq: + logger.info( + f"Does not match. [eq {eq}] [field {f}] [field val {this_field_value}] != [api val {this_api_value}]" + ) + equality_results.append(eq) + else: + # Log a message if field_ndx does not exist + logger.info( + f"Index {field_ndx} out of range for 'values' in row. Max index is {len(row['values']) - 1}" + ) + logger.info( + f"Field '{f}' with value '{this_api_value}' at index '{field_ndx}' is missing from test tables 'values'." + ) + + if all(equality_results): + count(summary, "correct_fields") + else: + count(summary, "incorrect_fields") + + def api_check(json_test_tables): combined_summary = {"endpoints": 0, "correct_rows": 0, "incorrect_rows": 0} for endo in json_test_tables: count(combined_summary, "endpoints") endpoint = endo["endpoint"] - report_id = endo["report_id"] summary = {} - equality_results = [] logger.info(f"-------------------- {endpoint} --------------------") - - for row_ndx, row in enumerate(endo["rows"]): - count(summary, "total_rows") - - if False in equality_results: - count(combined_summary, "incorrect_rows") - else: - count(combined_summary, "correct_rows") - - equality_results = [] - - for field_ndx, f in enumerate(row["fields"]): - # logger.info(f"Checking /{endpoint} {report_id} {f}") - # logger.info(f"{get_api_values(endpoint, report_id, f)}") - api_values = get_api_values(endpoint, report_id, f) - this_api_value = api_values[row_ndx] - - # Check if field_ndx exists in row["values"] - if field_ndx < len(row["values"]): - this_field_value = row["values"][field_ndx] - eq = check_equality(this_field_value, this_api_value) - if not eq: - logger.info( - f"Does not match. [eq {eq}] [field {f}] [field val {this_field_value}] != [api val {this_api_value}]" - ) - equality_results.append(eq) - else: - # Log a message if field_ndx does not exist - logger.info( - f"Index {field_ndx} out of range for 'values' in row. Max index is {len(row['values']) - 1}" - ) - logger.info( - f"Field '{f}' with value '{this_api_value}' at index '{field_ndx}' is missing from test tables 'values'." - ) - - if all(equality_results): - count(summary, "correct_fields") - else: - count(summary, "incorrect_fields") + process_singletons(endo, summary) + process_rows(endo, combined_summary, summary) logger.info(summary) combined_summary = combine_counts(combined_summary, summary) @@ -252,7 +267,7 @@ def api_check(json_test_tables): def run_end_to_end(user, audit_header, result): """Attempts to migrate the given audit""" try: - sac = setup_sac(user, audit_header) + sac, gen_api_data = setup_sac(user, audit_header) if sac.general_information["audit_type"] == "alternative-compliance-engagement": logger.info( @@ -271,6 +286,13 @@ def run_end_to_end(user, audit_header, result): (_, json, _) = builder_loader(fun, section) json_test_tables.append(json) + # Append total amount expended to general table checker + gen_api_data["singletons"]["total_amount_expended"] = sac.federal_awards[ + "FederalAwards" + ]["total_amount_expended"] + + json_test_tables.append(gen_api_data) + record_dummy_pdf_object(sac, user) step_through_certifications(sac) diff --git a/backend/census_historical_migration/historic_data_loader.py b/backend/census_historical_migration/historic_data_loader.py index e36d4dce4e..1c8a8e94bb 100644 --- a/backend/census_historical_migration/historic_data_loader.py +++ b/backend/census_historical_migration/historic_data_loader.py @@ -1,6 +1,6 @@ import logging from .models import ELECAUDITHEADER as AuditHeader -from .workbooklib.end_to_end_core import run_end_to_end +from .end_to_end_core import run_end_to_end from django.contrib.auth import get_user_model from django.core.paginator import Paginator @@ -51,7 +51,7 @@ def log_results(result_log, error_count, total_count): logger.info("********* Loader Summary ***************") for k, v in result_log.items(): - logger.info(k, v) + logger.info(f"{k}, {v}") logger.info("-------------------") logger.info(f"{error_count} errors out of {total_count}") @@ -67,7 +67,7 @@ def create_or_get_user(): if users: user = users.first() else: - logger.info("Creating user", user_email, user_name) + logger.info("Creating user %s %s", user_email, user_name) user = User(username=user_name, email=user_email) user.save() diff --git a/backend/census_historical_migration/management/commands/csv_to_postgres.py b/backend/census_historical_migration/management/commands/csv_to_postgres.py index ee5f3db03f..6a13ecb6e2 100644 --- a/backend/census_historical_migration/management/commands/csv_to_postgres.py +++ b/backend/census_historical_migration/management/commands/csv_to_postgres.py @@ -87,16 +87,16 @@ def process_csv_files(self, folder, chunk_size): def display_row_counts(self, models): for mdl in models: row_count = mdl.objects.all().count() - logger.info(f"{row_count} in ", mdl) + logger.info(f"{row_count} in {mdl}") def delete_data(self): for mdl in census_to_gsafac_models: - logger.info("Deleting ", mdl) + logger.info("Deleting %s", mdl) mdl.objects.all().delete() def sample_data(self): for mdl in census_to_gsafac_models: - logger.info("Sampling ", mdl) + logger.info("Sampling %s", mdl) rows = mdl.objects.all()[:1] for row in rows: for col in mdl._meta.fields: @@ -116,11 +116,11 @@ def get_s3_object(self, bucket_name, key, model_obj): return file def get_model_name(self, name): - logger.info("Processing ", name) + logger.info("Processing %s", name) file_name = name.split("/")[-1].split(".")[0] for model_name in census_to_gsafac_model_names: if file_name.lower().startswith(model_name): - logger.info("model_name = ", model_name) + logger.info("model_name = %s", model_name) return model_name logger.error("Could not find a matching model for ", name) return None @@ -139,5 +139,5 @@ def load_data(self, file, model_obj, chunk_size): obj = model_obj(**row) obj.save() rows_loaded += df.shape[0] - logger.info(f"Loaded {rows_loaded} rows in ", model_obj) + logger.info(f"Loaded {rows_loaded} rows in {model_obj}") return None diff --git a/backend/census_historical_migration/management/commands/historic_data_migrator.py b/backend/census_historical_migration/management/commands/historic_data_migrator.py index 0e8cfee65a..449858a1c4 100644 --- a/backend/census_historical_migration/management/commands/historic_data_migrator.py +++ b/backend/census_historical_migration/management/commands/historic_data_migrator.py @@ -11,7 +11,7 @@ create_or_get_user, log_results, ) -from census_historical_migration.workbooklib.end_to_end_core import run_end_to_end +from census_historical_migration.end_to_end_core import run_end_to_end from django.conf import settings logger = logging.getLogger(__name__) diff --git a/backend/census_historical_migration/sac_general_lib/audit_information.py b/backend/census_historical_migration/sac_general_lib/audit_information.py index 8451ca0bef..2fe829b16a 100644 --- a/backend/census_historical_migration/sac_general_lib/audit_information.py +++ b/backend/census_historical_migration/sac_general_lib/audit_information.py @@ -1,5 +1,7 @@ import re +from django.conf import settings +from ..api_test_helpers import extract_api_data from ..transforms.xform_string_to_int import string_to_int from ..transforms.xform_string_to_bool import string_to_bool from ..exception_utils import DataMigrationError @@ -19,7 +21,7 @@ def xform_apply_default_thresholds(value): str_value = string_to_string(value) if str_value == "": # FIXME-MSHD: This is a transformation that we may want to record - return -1 + return settings.GSA_MIGRATION_INT return string_to_int(str_value) @@ -163,7 +165,8 @@ def audit_information(audit_header): } audit_info["agencies"] = list(agencies_prefixes) - # Validate against the schema audit.validators.validate_audit_information_json(audit_info) - return audit_info + api_data = extract_api_data(mappings, audit_info) + + return (audit_info, api_data) diff --git a/backend/census_historical_migration/sac_general_lib/general_information.py b/backend/census_historical_migration/sac_general_lib/general_information.py index 64b4ae9060..43116c4780 100644 --- a/backend/census_historical_migration/sac_general_lib/general_information.py +++ b/backend/census_historical_migration/sac_general_lib/general_information.py @@ -2,6 +2,7 @@ import audit.validators from datetime import timedelta +from ..api_test_helpers import extract_api_data from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..transforms.xform_remove_hyphen_and_pad_zip import xform_remove_hyphen_and_pad_zip from ..transforms.xform_string_to_string import string_to_string @@ -240,7 +241,6 @@ def general_information(audit_header): general_information = create_json_from_db_object(audit_header, mappings) - # List of transformation functions transformations = [ xform_auditee_fiscal_period_start, xform_auditee_fiscal_period_end, @@ -249,14 +249,14 @@ def general_information(audit_header): xform_audit_type, ] - # Apply transformations for transform in transformations: if transform == xform_country: general_information = transform(general_information, audit_header) else: general_information = transform(general_information) - # verify that the created object validates against the schema audit.validators.validate_general_information_complete_json(general_information) - return general_information + api_data = extract_api_data(mappings, general_information) + + return (general_information, api_data) diff --git a/backend/census_historical_migration/sac_general_lib/sac_creator.py b/backend/census_historical_migration/sac_general_lib/sac_creator.py index 80ec1bc555..1a0e8055f6 100644 --- a/backend/census_historical_migration/sac_general_lib/sac_creator.py +++ b/backend/census_historical_migration/sac_general_lib/sac_creator.py @@ -3,6 +3,7 @@ from django.apps import apps from django.conf import settings +from ..api_test_helpers import generate_dissemination_test_table from ..exception_utils import DataMigrationError from .general_information import ( general_information, @@ -43,12 +44,13 @@ def setup_sac(user, audit_header): if exists: exists.delete() - general_info = general_information(audit_header) + general_info, gen_api_data = general_information(audit_header) + audit_info, audit_api_data = audit_information(audit_header) sac = SingleAuditChecklist.objects.create( submitted_by=user, general_information=general_info, - audit_information=audit_information(audit_header), + audit_information=audit_info, audit_type=general_info["audit_type"], ) @@ -80,4 +82,9 @@ def setup_sac(user, audit_header): sac.data_source = settings.CENSUS_DATA_SOURCE sac.save() logger.info("Created single audit checklist %s", sac) - return sac + + table = generate_dissemination_test_table(audit_header, "general") + table["singletons"].update(gen_api_data) + table["singletons"].update(audit_api_data) + + return (sac, table) diff --git a/backend/census_historical_migration/test_federal_awards_xforms.py b/backend/census_historical_migration/test_federal_awards_xforms.py index 3d6dacbed8..096118ece7 100644 --- a/backend/census_historical_migration/test_federal_awards_xforms.py +++ b/backend/census_historical_migration/test_federal_awards_xforms.py @@ -286,10 +286,10 @@ def test_passthrough_award_N_unexpected_amount(self): xform_populate_default_passthrough_amount(audits) def test_passthrough_award_Y_empty_amount(self): - """Test the function raises DataMigrationError when passthrough award Y audit with empty amount.""" + """Test for default value when passthrough award Y audit with empty amount.""" audits = [self.MockAudit(PASSTHROUGHAWARD="Y", PASSTHROUGHAMOUNT="")] - with self.assertRaises(DataMigrationError): - xform_populate_default_passthrough_amount(audits) + expected = [str(settings.GSA_MIGRATION_INT)] + self.assertEqual(xform_populate_default_passthrough_amount(audits), expected) class TestCFDAFunctions(SimpleTestCase): diff --git a/backend/census_historical_migration/workbooklib/additional_eins.py b/backend/census_historical_migration/workbooklib/additional_eins.py index 6b1a8994e9..88b5217e88 100644 --- a/backend/census_historical_migration/workbooklib/additional_eins.py +++ b/backend/census_historical_migration/workbooklib/additional_eins.py @@ -1,10 +1,10 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..transforms.xform_string_to_string import ( string_to_string, ) from ..workbooklib.excel_creation_utils import ( map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import ( @@ -55,7 +55,7 @@ def xform_remove_trailing_decimal_zero(value): def _get_eins(dbkey, year): - return Eins.objects.filter(DBKEY=dbkey, AUDITYEAR=year) + return Eins.objects.filter(DBKEY=dbkey, AUDITYEAR=year).exclude(EIN="") def generate_additional_eins(audit_header, outfile): @@ -75,5 +75,5 @@ def generate_additional_eins(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "additional_eins", mappings, addl_eins ) - table["singletons"]["auditee_uei"] = uei + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/additional_ueis.py b/backend/census_historical_migration/workbooklib/additional_ueis.py index 1f31730b65..9fbbb3dae4 100644 --- a/backend/census_historical_migration/workbooklib/additional_ueis.py +++ b/backend/census_historical_migration/workbooklib/additional_ueis.py @@ -1,7 +1,7 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..workbooklib.excel_creation_utils import ( map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import ( @@ -24,7 +24,7 @@ def _get_ueis(dbkey, year): - return Ueis.objects.filter(DBKEY=dbkey, AUDITYEAR=year) + return Ueis.objects.filter(DBKEY=dbkey, AUDITYEAR=year).exclude(UEI="") def generate_additional_ueis(audit_header, outfile): @@ -44,5 +44,5 @@ def generate_additional_ueis(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "additional_ueis", mappings, additional_ueis ) - table["singletons"]["auditee_uei"] = uei + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/corrective_action_plan.py b/backend/census_historical_migration/workbooklib/corrective_action_plan.py index d5333638f7..0c1c9f350d 100644 --- a/backend/census_historical_migration/workbooklib/corrective_action_plan.py +++ b/backend/census_historical_migration/workbooklib/corrective_action_plan.py @@ -1,7 +1,7 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..workbooklib.excel_creation_utils import ( map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import ( @@ -54,5 +54,5 @@ def generate_corrective_action_plan(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "corrective_action_plans", mappings, captexts ) - table["singletons"]["auditee_uei"] = uei + return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/excel_creation_utils.py b/backend/census_historical_migration/workbooklib/excel_creation_utils.py index 7e6b1746f8..f14e3aa68a 100644 --- a/backend/census_historical_migration/workbooklib/excel_creation_utils.py +++ b/backend/census_historical_migration/workbooklib/excel_creation_utils.py @@ -9,11 +9,7 @@ DataMigrationError, DataMigrationValueError, ) -from ..base_field_maps import WorkbookFieldInDissem from ..workbooklib.templates import sections_to_template_paths -from ..sac_general_lib.report_id_generator import ( - xform_dbkey_to_report_id, -) from ..models import ( ELECAUDITS as Audits, ELECAUDITHEADER as AuditHeader, @@ -204,36 +200,6 @@ def get_template_name_for_section(section): ) -def generate_dissemination_test_table(audit_header, api_endpoint, mappings, objects): - """Generates a test table for verifying the API queries results.""" - table = {"rows": list(), "singletons": dict()} - table["endpoint"] = api_endpoint - table["report_id"] = xform_dbkey_to_report_id(audit_header) - - for o in objects: - test_obj = {} - test_obj["fields"] = [] - test_obj["values"] = [] - for m in mappings: - # What if we only test non-null values? - raw_value = getattr(o, m.in_db, None) - attribute_value = apply_conversion_function(raw_value, m.default, m.type) - if (attribute_value is not None) and (attribute_value != ""): - if m.in_dissem == WorkbookFieldInDissem: - # print(f'in_sheet {m.in_sheet} <- {attribute_value}') - test_obj["fields"].append(m.in_sheet) - # The typing must be applied here as well, as in the case of - # type_requirement, it alphabetizes the value... - test_obj["values"].append(m.type(attribute_value)) - else: - # print(f'in_dissem {m.in_dissem} <- {attribute_value}') - test_obj["fields"].append(m.in_dissem) - test_obj["values"].append(m.type(attribute_value)) - - table["rows"].append(test_obj) - return table - - def get_audits(dbkey, year): """Returns Audits records for the given dbkey and audit year.""" return Audits.objects.filter(DBKEY=dbkey, AUDITYEAR=year).order_by("ID") diff --git a/backend/census_historical_migration/workbooklib/federal_awards.py b/backend/census_historical_migration/workbooklib/federal_awards.py index 93cdb5d391..f4b175ba31 100644 --- a/backend/census_historical_migration/workbooklib/federal_awards.py +++ b/backend/census_historical_migration/workbooklib/federal_awards.py @@ -1,5 +1,5 @@ import re - +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..exception_utils import DataMigrationError from ..transforms.xform_string_to_string import ( @@ -11,7 +11,6 @@ get_ranges, set_workbook_uei, map_simple_columns, - generate_dissemination_test_table, set_range, ) from ..base_field_maps import ( @@ -286,9 +285,7 @@ def xform_populate_default_passthrough_amount(audits): else: # FIXME -MSHD: Is this what we want to do? # Changing to settings.GSA_MIGRATION will require an update to the field type in dissemination model. - raise DataMigrationError( - "Missing passthrough amount.", "missing_passthrough_amount" - ) + passthrough_amounts.append(str(settings.GSA_MIGRATION_INT)) else: if not amount or amount == "0": passthrough_amounts.append("") @@ -388,74 +385,72 @@ def generate_federal_awards(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "federal_awards", mappings, audits ) - award_counter = 1 - filtered_mappings = [ - mapping - for mapping in mappings - if mapping.in_sheet - in [ - "additional_award_identification", - "federal_agency_prefix", - "three_digit_extension", - "passthrough_name", - "passthrough_identifying_number", - "subrecipient_amount", - "loan_balance_at_audit_period_end", + if audits: + award_counter = 1 + filtered_mappings = [ + mapping + for mapping in mappings + if mapping.in_sheet + in [ + "additional_award_identification", + "federal_agency_prefix", + "three_digit_extension", + "passthrough_name", + "passthrough_identifying_number", + "subrecipient_amount", + "loan_balance_at_audit_period_end", + ] ] - ] - ranges = get_ranges(filtered_mappings, audits) - prefixes = get_range_values(ranges, "federal_agency_prefix") - extensions = get_range_values(ranges, "three_digit_extension") - - for ( - award, - prefix, - extension, - additional_identification, - cluster_name, - other_cluster_name, - state_cluster_name, - passthrough_amount, - loan_balance, - ) in zip( - table["rows"], - prefixes, - extensions, - additional_award_identifications, - cluster_names, - other_cluster_names, - state_cluster_names, - passthrough_amounts, - loan_balances, - ): - # Function to update or append field/value - def update_or_append_field(field_name, field_value): - if field_name in award["fields"]: - index = award["fields"].index(field_name) - logger.info( - f"Updating {field_name} from {award['values'][index]} to {field_value}" - ) - award["values"][index] = field_value - else: - award["fields"].append(field_name) - award["values"].append(field_value) - - # Update or append new field-value pairs - update_or_append_field("federal_agency_prefix", prefix) - update_or_append_field("federal_award_extension", extension) - update_or_append_field("award_reference", f"AWARD-{award_counter:04}") - update_or_append_field( - "additional_award_identification", additional_identification - ) - update_or_append_field("cluster_name", cluster_name) - update_or_append_field("other_cluster_name", other_cluster_name) - update_or_append_field("state_cluster_name", state_cluster_name) - update_or_append_field("passthrough_amount", passthrough_amount) - update_or_append_field("loan_balance", loan_balance) - - award_counter += 1 - - table["singletons"]["auditee_uei"] = uei - table["singletons"]["total_amount_expended"] = total + ranges = get_ranges(filtered_mappings, audits) + prefixes = get_range_values(ranges, "federal_agency_prefix") + extensions = get_range_values(ranges, "three_digit_extension") + + for ( + award, + prefix, + extension, + additional_identification, + cluster_name, + other_cluster_name, + state_cluster_name, + passthrough_amount, + loan_balance, + ) in zip( + table["rows"], + prefixes, + extensions, + additional_award_identifications, + cluster_names, + other_cluster_names, + state_cluster_names, + passthrough_amounts, + loan_balances, + ): + # Function to update or append field/value + def update_or_append_field(field_name, field_value): + if field_name in award["fields"]: + index = award["fields"].index(field_name) + logger.info( + f"Updating {field_name} from {award['values'][index]} to {field_value}" + ) + award["values"][index] = field_value + else: + award["fields"].append(field_name) + award["values"].append(field_value) + + # Update or append new field-value pairs + update_or_append_field("federal_agency_prefix", prefix) + update_or_append_field("federal_award_extension", extension) + update_or_append_field("award_reference", f"AWARD-{award_counter:04}") + update_or_append_field( + "additional_award_identification", additional_identification + ) + update_or_append_field("cluster_name", cluster_name) + update_or_append_field("other_cluster_name", other_cluster_name) + update_or_append_field("state_cluster_name", state_cluster_name) + update_or_append_field("passthrough_amount", passthrough_amount) + update_or_append_field("loan_balance", loan_balance) + + award_counter += 1 return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/findings.py b/backend/census_historical_migration/workbooklib/findings.py index a9ffb512f8..cd3cd7a4f2 100644 --- a/backend/census_historical_migration/workbooklib/findings.py +++ b/backend/census_historical_migration/workbooklib/findings.py @@ -1,3 +1,4 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..transforms.xform_string_to_string import ( string_to_string, @@ -5,7 +6,6 @@ from ..workbooklib.excel_creation_utils import ( get_audits, map_simple_columns, - generate_dissemination_test_table, set_range, set_workbook_uei, ) @@ -154,8 +154,9 @@ def generate_findings(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "findings", mappings, findings ) - for obj, ar in zip(table["rows"], award_references): - obj["fields"].append("award_reference") - obj["values"].append(ar) + if findings: + for obj, ar in zip(table["rows"], award_references): + obj["fields"].append("award_reference") + obj["values"].append(ar) return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/findings_text.py b/backend/census_historical_migration/workbooklib/findings_text.py index e015392537..3adf7c2426 100644 --- a/backend/census_historical_migration/workbooklib/findings_text.py +++ b/backend/census_historical_migration/workbooklib/findings_text.py @@ -1,7 +1,7 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..workbooklib.excel_creation_utils import ( map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import ( @@ -55,6 +55,5 @@ def generate_findings_text(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "findings_text", mappings, findings_texts ) - table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/notes_to_sefa.py b/backend/census_historical_migration/workbooklib/notes_to_sefa.py index f5bcd73972..4374e56841 100644 --- a/backend/census_historical_migration/workbooklib/notes_to_sefa.py +++ b/backend/census_historical_migration/workbooklib/notes_to_sefa.py @@ -1,5 +1,5 @@ from django.conf import settings - +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..exception_utils import DataMigrationError from ..transforms.xform_string_to_string import string_to_string @@ -7,7 +7,6 @@ from ..workbooklib.excel_creation_utils import ( set_range, map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import SheetFieldMap @@ -156,13 +155,15 @@ def generate_notes_to_sefa(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "notes_to_sefa", mappings, notes ) + table["singletons"] = dict() table["singletons"]["accounting_policies"] = policies_content table["singletons"]["is_minimis_rate_used"] = is_minimis_rate_used table["singletons"]["rate_explained"] = rate_content table["singletons"]["auditee_uei"] = uei - for obj, ar in zip(table["rows"], contains_chart_or_tables): - obj["fields"].append("contains_chart_or_table") - obj["values"].append(ar) + if notes: + for obj, ar in zip(table["rows"], contains_chart_or_tables): + obj["fields"].append("contains_chart_or_table") + obj["values"].append(ar) return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/post_upload_utils.py b/backend/census_historical_migration/workbooklib/post_upload_utils.py index 1ef9a6ed61..e26dcdb63f 100644 --- a/backend/census_historical_migration/workbooklib/post_upload_utils.py +++ b/backend/census_historical_migration/workbooklib/post_upload_utils.py @@ -83,7 +83,7 @@ def record_dummy_pdf_object(this_sac, this_user): this_sac.save() -def _post_upload_workbook(this_sac, section, xlsx_file): +def post_upload_workbook(this_sac, section, xlsx_file): """Upload a workbook for this SAC.""" audit_data = extract_mapping[section](xlsx_file, is_gsa_migration=True) diff --git a/backend/census_historical_migration/workbooklib/secondary_auditors.py b/backend/census_historical_migration/workbooklib/secondary_auditors.py index ac74d2f59d..f67c5ec1d8 100644 --- a/backend/census_historical_migration/workbooklib/secondary_auditors.py +++ b/backend/census_historical_migration/workbooklib/secondary_auditors.py @@ -1,8 +1,8 @@ +from ..api_test_helpers import generate_dissemination_test_table from ..transforms.xform_retrieve_uei import xform_retrieve_uei from ..transforms.xform_remove_hyphen_and_pad_zip import xform_remove_hyphen_and_pad_zip from ..workbooklib.excel_creation_utils import ( map_simple_columns, - generate_dissemination_test_table, set_workbook_uei, ) from ..base_field_maps import SheetFieldMap @@ -85,6 +85,5 @@ def generate_secondary_auditors(audit_header, outfile): table = generate_dissemination_test_table( audit_header, "secondary_auditors", mappings, secondary_auditors ) - table["singletons"]["auditee_uei"] = uei return (wb, table) diff --git a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py index 70ddce180d..0866c019a0 100644 --- a/backend/census_historical_migration/workbooklib/workbook_builder_loader.py +++ b/backend/census_historical_migration/workbooklib/workbook_builder_loader.py @@ -1,6 +1,6 @@ from ..workbooklib.workbook_builder import generate_workbook from ..workbooklib.post_upload_utils import ( - _post_upload_workbook, + post_upload_workbook, ) import logging @@ -21,7 +21,7 @@ def _loader(workbook_generator, section): ) if user: - _post_upload_workbook(sac, section, excel_file) + post_upload_workbook(sac, section, excel_file) else: raise Exception("User must be provided to upload workbook") diff --git a/backend/config/settings.py b/backend/config/settings.py index fae719f140..dcb3129169 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -542,3 +542,4 @@ REGEX_THREE_DIGIT_EXTENSION = r"^[0-9]{3}[A-Za-z]{0,1}$" REGEX_U_EXTENSION = r"^U[0-9]{2}$" GSA_MIGRATION = "GSA_MIGRATION" +GSA_MIGRATION_INT = -999999999 From 84b08a10224a933d4bdda0456ab2b75cbdc829de Mon Sep 17 00:00:00 2001 From: Tim Ballard <1425377+timoballard@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:27:02 -0600 Subject: [PATCH 2/2] reduce summary report limit to 500 (#3084) --- backend/config/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/config/settings.py b/backend/config/settings.py index dcb3129169..8a73d621ee 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -534,7 +534,7 @@ CENSUS_DATA_SOURCE = "CENSUS" GSA_MIGRATION = "GSA_MIGRATION" # There is a copy of `GSA_MIGRATION` in Base.libsonnet. If you change it here, change it there too. DOLLAR_THRESHOLD = 750000 -SUMMARY_REPORT_DOWNLOAD_LIMIT = 1000 +SUMMARY_REPORT_DOWNLOAD_LIMIT = 500 # A version of these regexes also exists in Base.libsonnet REGEX_ALN_PREFIX = r"^([0-9]{2})$"