diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx new file mode 100644 index 0000000000..871257262a Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx differ diff --git a/backend/audit/intakelib/checks/check_additional_award_identification_present.py b/backend/audit/intakelib/checks/check_additional_award_identification_present.py index 80c038faf6..2b72a4f785 100644 --- a/backend/audit/intakelib/checks/check_additional_award_identification_present.py +++ b/backend/audit/intakelib/checks/check_additional_award_identification_present.py @@ -8,7 +8,7 @@ REGEX_RD_EXTENSION, REGEX_U_EXTENSION, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py index ed580ba2d5..d594064edd 100644 --- a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py +++ b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py index 6ef76ed313..1bf908002e 100644 --- a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py +++ b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py @@ -4,7 +4,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py index cef39db826..fdbedcb26b 100644 --- a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py +++ b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py @@ -2,7 +2,7 @@ from audit.intakelib.intermediate_representation import ( get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_cluster_total.py b/backend/audit/intakelib/checks/check_cluster_total.py index 53e3945f19..fadeb51fff 100644 --- a/backend/audit/intakelib/checks/check_cluster_total.py +++ b/backend/audit/intakelib/checks/check_cluster_total.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_eins_are_not_empty.py b/backend/audit/intakelib/checks/check_eins_are_not_empty.py index 2ad2ef6b35..6c0254aef2 100644 --- a/backend/audit/intakelib/checks/check_eins_are_not_empty.py +++ b/backend/audit/intakelib/checks/check_eins_are_not_empty.py @@ -1,6 +1,6 @@ from django.core.exceptions import ValidationError import logging -from .util import get_missing_value_errors +from audit.intakelib.common import get_missing_value_errors logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py similarity index 95% rename from backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py rename to backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py index d28e043d53..449bea23fe 100644 --- a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py +++ b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_federal_program_total.py b/backend/audit/intakelib/checks/check_federal_program_total.py index d7389f418f..77e52e394f 100644 --- a/backend/audit/intakelib/checks/check_federal_program_total.py +++ b/backend/audit/intakelib/checks/check_federal_program_total.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py index afc8a12820..c9f631929f 100644 --- a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py +++ b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py @@ -4,7 +4,12 @@ from audit.intakelib.intermediate_representation import ( get_range_by_name, ) -from .util import get_message, build_cell_error_tuple, appears_empty, is_value_na +from audit.intakelib.common import ( + get_message, + build_cell_error_tuple, + appears_empty, + is_value_marked_na, +) logger = logging.getLogger(__name__) @@ -22,7 +27,7 @@ def prior_references_pattern(ir): for index, prior_reference in enumerate(prior_references["values"]): if ( not appears_empty(prior_reference) - and (not is_value_na(prior_reference)) + and (not is_value_marked_na(prior_reference)) and (not re.match(PRIOR_REFERENCES_REGEX, str(prior_reference))) ): errors.append( diff --git a/backend/audit/intakelib/checks/check_findings_grid_validation.py b/backend/audit/intakelib/checks/check_findings_grid_validation.py index 7a910bddef..d22f0eebf6 100644 --- a/backend/audit/intakelib/checks/check_findings_grid_validation.py +++ b/backend/audit/intakelib/checks/check_findings_grid_validation.py @@ -2,7 +2,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple # Modified Opinion # Other Matters diff --git a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py index 922b5d7997..e4ba189044 100644 --- a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py +++ b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py @@ -3,7 +3,7 @@ from audit.intakelib.intermediate_representation import ( extract_workbook_as_ir, ) -from .util import get_names_of_all_ranges +from audit.intakelib.common import get_names_of_all_ranges from audit.fixtures.excel import FORM_SECTIONS from audit.fixtures.excel import ( ADDITIONAL_UEIS_TEMPLATE, diff --git a/backend/audit/intakelib/checks/check_is_right_workbook.py b/backend/audit/intakelib/checks/check_is_right_workbook.py index 8ef87f12a3..81d7bc5a2e 100644 --- a/backend/audit/intakelib/checks/check_is_right_workbook.py +++ b/backend/audit/intakelib/checks/check_is_right_workbook.py @@ -4,7 +4,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_loan_balance_entries.py b/backend/audit/intakelib/checks/check_loan_balance_entries.py index 96c33660eb..7b2be52750 100644 --- a/backend/audit/intakelib/checks/check_loan_balance_entries.py +++ b/backend/audit/intakelib/checks/check_loan_balance_entries.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_loan_balance_present.py b/backend/audit/intakelib/checks/check_loan_balance_present.py index f661f9b223..33e882dcf2 100644 --- a/backend/audit/intakelib/checks/check_loan_balance_present.py +++ b/backend/audit/intakelib/checks/check_loan_balance_present.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_look_for_empty_rows.py b/backend/audit/intakelib/checks/check_look_for_empty_rows.py index 437e0e2a9f..a7a805265e 100644 --- a/backend/audit/intakelib/checks/check_look_for_empty_rows.py +++ b/backend/audit/intakelib/checks/check_look_for_empty_rows.py @@ -2,7 +2,7 @@ import logging from audit.intakelib.intermediate_representation import ranges_to_rows, appears_empty -from .util import get_range_start_row +from audit.intakelib.common import get_range_start_row logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_missing_required_fields.py b/backend/audit/intakelib/checks/check_missing_required_fields.py index 3785bf3bf7..8f2d37a7cc 100644 --- a/backend/audit/intakelib/checks/check_missing_required_fields.py +++ b/backend/audit/intakelib/checks/check_missing_required_fields.py @@ -1,6 +1,6 @@ from django.core.exceptions import ValidationError import logging -from .util import get_missing_value_errors +from audit.intakelib.common import get_missing_value_errors from audit.fixtures.excel import FORM_SECTIONS logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_no_major_program_no_type.py b/backend/audit/intakelib/checks/check_no_major_program_no_type.py index f6dbc639d3..a11dd02f5a 100644 --- a/backend/audit/intakelib/checks/check_no_major_program_no_type.py +++ b/backend/audit/intakelib/checks/check_no_major_program_no_type.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_no_repeat_findings.py b/backend/audit/intakelib/checks/check_no_repeat_findings.py index efabc06e35..7f45a958f8 100644 --- a/backend/audit/intakelib/checks/check_no_repeat_findings.py +++ b/backend/audit/intakelib/checks/check_no_repeat_findings.py @@ -1,6 +1,10 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple, is_value_na +from audit.intakelib.common import ( + get_message, + build_cell_error_tuple, + is_value_marked_na, +) logger = logging.getLogger(__name__) @@ -13,7 +17,7 @@ def no_repeat_findings(ir): for ndx, (is_rep, prior) in enumerate( zip(repeat_prior_reference["values"], prior_references["values"]) ): - if (is_rep == "N") and (not is_value_na(prior)): + if (is_rep == "N") and (not is_value_marked_na(prior)): errors.append( build_cell_error_tuple( ir, @@ -22,7 +26,7 @@ def no_repeat_findings(ir): get_message("check_no_repeat_findings_when_n"), ) ) - elif (is_rep == "Y") and ((not prior) or (is_value_na(prior))): + elif (is_rep == "Y") and ((not prior) or (is_value_marked_na(prior))): errors.append( build_cell_error_tuple( ir, diff --git a/backend/audit/intakelib/checks/check_other_cluster_names.py b/backend/audit/intakelib/checks/check_other_cluster_names.py index 85a0ba6bd1..57de4dd0e6 100644 --- a/backend/audit/intakelib/checks/check_other_cluster_names.py +++ b/backend/audit/intakelib/checks/check_other_cluster_names.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py index de11e8ab87..c63a9c68dd 100644 --- a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py +++ b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_sequential_award_numbers.py b/backend/audit/intakelib/checks/check_sequential_award_numbers.py index 06bae0bbd1..850f3f26c8 100644 --- a/backend/audit/intakelib/checks/check_sequential_award_numbers.py +++ b/backend/audit/intakelib/checks/check_sequential_award_numbers.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple import re logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_state_cluster_names.py b/backend/audit/intakelib/checks/check_state_cluster_names.py index f3f692e5f2..8e984eebbc 100644 --- a/backend/audit/intakelib/checks/check_state_cluster_names.py +++ b/backend/audit/intakelib/checks/check_state_cluster_names.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_total_amount_expended.py b/backend/audit/intakelib/checks/check_total_amount_expended.py index e9a041bed9..6e500a490f 100644 --- a/backend/audit/intakelib/checks/check_total_amount_expended.py +++ b/backend/audit/intakelib/checks/check_total_amount_expended.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_uei_exists.py b/backend/audit/intakelib/checks/check_uei_exists.py index 0648d2679f..bc88ec839f 100644 --- a/backend/audit/intakelib/checks/check_uei_exists.py +++ b/backend/audit/intakelib/checks/check_uei_exists.py @@ -1,6 +1,10 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import list_contains_non_null_values, get_message, build_range_error_tuple +from audit.intakelib.common import ( + list_contains_non_null_values, + get_message, + build_range_error_tuple, +) logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_y_or_n__fields.py b/backend/audit/intakelib/checks/check_y_or_n__fields.py index fe40db7d24..0af2674761 100644 --- a/backend/audit/intakelib/checks/check_y_or_n__fields.py +++ b/backend/audit/intakelib/checks/check_y_or_n__fields.py @@ -1,5 +1,5 @@ import logging -from .util import invalid_y_or_n_entry +from audit.intakelib.common import invalid_y_or_n_entry from audit.fixtures.excel import FORM_SECTIONS logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/runners.py b/backend/audit/intakelib/checks/runners.py index 22aa490255..34c71bd427 100644 --- a/backend/audit/intakelib/checks/runners.py +++ b/backend/audit/intakelib/checks/runners.py @@ -29,7 +29,7 @@ from .check_federal_program_total import federal_program_total_is_correct from .check_cluster_total import cluster_total_is_correct from .check_total_amount_expended import total_amount_expended_is_correct -from .check_federal_award_passed_passed_through_optional import ( +from .check_federal_award_passed_through_optional import ( federal_award_amount_passed_through_optional, ) from .check_cardinality_of_passthrough_names_and_ids import ( diff --git a/backend/audit/intakelib/common/__init__.py b/backend/audit/intakelib/common/__init__.py new file mode 100644 index 0000000000..665ee1f767 --- /dev/null +++ b/backend/audit/intakelib/common/__init__.py @@ -0,0 +1,14 @@ +# flake8: noqa: F401 +from .util import ( + get_message, + build_cell_error_tuple, + appears_empty, + is_value_marked_na, + get_names_of_all_ranges, + get_range_start_row, + get_missing_value_errors, + invalid_y_or_n_entry, + safe_int_conversion, + list_contains_non_null_values, + build_range_error_tuple, +) diff --git a/backend/audit/intakelib/checks/error_messages.py b/backend/audit/intakelib/common/error_messages.py similarity index 98% rename from backend/audit/intakelib/checks/error_messages.py rename to backend/audit/intakelib/common/error_messages.py index 5479c0bc2b..e968c6c6cb 100644 --- a/backend/audit/intakelib/checks/error_messages.py +++ b/backend/audit/intakelib/common/error_messages.py @@ -55,4 +55,5 @@ "check_federal_award_amount_passed_through_not_allowed": "When Federal Award Passed Through is N, Amount Passed Through must be empty", "check_loan_balance": "The loan balance is currently set to {}. It should either be a positive number, N/A, or left empty", "check_cardinality_of_passthrough_names_and_ids": "You used a | (bar character) to indicate multiple passthrough names and IDs; you must provide equal numbers of names and IDs. You provided {} name{} and {} ID{}", + "check_integer_values": "{} is not a valid integer", } diff --git a/backend/audit/intakelib/checks/util.py b/backend/audit/intakelib/common/util.py similarity index 66% rename from backend/audit/intakelib/checks/util.py rename to backend/audit/intakelib/common/util.py index 4aa9b3d322..42ebfba623 100644 --- a/backend/audit/intakelib/checks/util.py +++ b/backend/audit/intakelib/common/util.py @@ -2,7 +2,9 @@ import logging from audit.intakelib.intermediate_representation import ( get_range_by_name, + replace_range_by_name, ) +from django.core.exceptions import ValidationError logger = logging.getLogger(__name__) @@ -59,7 +61,7 @@ def appears_empty(v): return (v is None) or (str(v).strip() == "") -def is_value_na(v): +def is_value_marked_na(v): value = str(v).strip() return value == "N/A" @@ -113,3 +115,38 @@ def get_names_of_all_ranges(data): if "name" in range_item: names.append(range_item["name"]) return names + + +def safe_int_conversion(ir, range_name, other_values_allowed=None): + range_data = get_range_by_name(ir, range_name) + errors = [] + new_values = [] + if range_data: + for index, value in enumerate(range_data["values"]): + try: + float_value = float(value) + if float_value.is_integer(): + new_values.append(int(float_value)) + else: + raise ValueError + except (ValueError, TypeError): + # If the value is None, we keep it. This is because some int fields are optional. + # For non optional fields, there is a check for missing required fields that will raise an error. + if (value is None) or ( + other_values_allowed and value in other_values_allowed + ): + new_values.append(value) + else: + errors.append( + build_cell_error_tuple( + ir, + range_data, + index, + get_message("check_integer_values").format(value), + ) + ) + if len(errors) > 0: + logger.info("Raising a validation error.") + raise ValidationError(errors) + new_ir = replace_range_by_name(ir, range_name, new_values) + return new_ir diff --git a/backend/audit/intakelib/mapping_additional_eins.py b/backend/audit/intakelib/mapping_additional_eins.py index 2f306ab19e..31180723e4 100644 --- a/backend/audit/intakelib/mapping_additional_eins.py +++ b/backend/audit/intakelib/mapping_additional_eins.py @@ -46,8 +46,8 @@ def extract_additional_eins(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_EINS) - run_all_additional_eins_checks(ir) xform_ir = run_all_additional_eins_transforms(ir) + run_all_additional_eins_checks(xform_ir) result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_additional_ueis.py b/backend/audit/intakelib/mapping_additional_ueis.py index ec3c65b88d..d58696a862 100644 --- a/backend/audit/intakelib/mapping_additional_ueis.py +++ b/backend/audit/intakelib/mapping_additional_ueis.py @@ -22,7 +22,7 @@ ) from .mapping_meta import meta_mapping - +from .transforms import run_all_additional_ueis_transforms from .checks import run_all_general_checks, run_all_additional_ueis_checks logger = logging.getLogger(__name__) @@ -43,8 +43,9 @@ def extract_additional_ueis(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_UEIS) - run_all_additional_ueis_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_additional_ueis_transforms(ir) + run_all_additional_ueis_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_audit_findings.py b/backend/audit/intakelib/mapping_audit_findings.py index a481605efa..6b29bd02b8 100644 --- a/backend/audit/intakelib/mapping_audit_findings.py +++ b/backend/audit/intakelib/mapping_audit_findings.py @@ -43,9 +43,9 @@ def extract_audit_findings(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE) - new_ir = run_all_audit_findings_transforms(ir) - run_all_audit_finding_checks(new_ir) - result = _extract_generic_data(new_ir, params) + xform_ir = run_all_audit_findings_transforms(ir) + run_all_audit_finding_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_audit_findings_text.py b/backend/audit/intakelib/mapping_audit_findings_text.py index dcc0887fff..a7817e8e3b 100644 --- a/backend/audit/intakelib/mapping_audit_findings_text.py +++ b/backend/audit/intakelib/mapping_audit_findings_text.py @@ -21,7 +21,7 @@ ) from .mapping_meta import meta_mapping - +from .transforms import run_all_audit_findings_text_transforms from .checks import run_all_general_checks, run_all_audit_findings_text_checks logger = logging.getLogger(__name__) @@ -42,8 +42,9 @@ def extract_audit_findings_text(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_TEXT) - run_all_audit_findings_text_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_audit_findings_text_transforms(ir) + run_all_audit_findings_text_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_corrective_action_plan.py b/backend/audit/intakelib/mapping_corrective_action_plan.py index 1068f784cb..82ae5b05ae 100644 --- a/backend/audit/intakelib/mapping_corrective_action_plan.py +++ b/backend/audit/intakelib/mapping_corrective_action_plan.py @@ -22,6 +22,7 @@ from .mapping_meta import meta_mapping from .checks import run_all_general_checks, run_all_corrective_action_plan_checks +from .transforms import run_all_corrective_action_plan_transforms logger = logging.getLogger(__name__) @@ -41,8 +42,9 @@ def extract_corrective_action_plan(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.CORRECTIVE_ACTION_PLAN) - run_all_corrective_action_plan_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_corrective_action_plan_transforms(ir) + run_all_corrective_action_plan_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_secondary_auditors.py b/backend/audit/intakelib/mapping_secondary_auditors.py index 02a3c31d73..acd99a94e3 100644 --- a/backend/audit/intakelib/mapping_secondary_auditors.py +++ b/backend/audit/intakelib/mapping_secondary_auditors.py @@ -18,7 +18,7 @@ from .mapping_meta import meta_mapping from .checks import run_all_general_checks, run_all_secondary_auditors_checks - +from .transforms import run_all_secondary_auditors_transforms from .intermediate_representation import ( extract_workbook_as_ir, _extract_generic_data, @@ -42,8 +42,9 @@ def extract_secondary_auditors(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.SECONDARY_AUDITORS) - run_all_secondary_auditors_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_secondary_auditors_transforms(ir) + run_all_secondary_auditors_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/transforms/__init__.py b/backend/audit/intakelib/transforms/__init__.py index e42e5a6f12..b59f5eb09f 100644 --- a/backend/audit/intakelib/transforms/__init__.py +++ b/backend/audit/intakelib/transforms/__init__.py @@ -3,5 +3,9 @@ run_all_notes_to_sefa_transforms, run_all_additional_eins_transforms, run_all_federal_awards_transforms, + run_all_additional_ueis_transforms, + run_all_audit_findings_text_transforms, run_all_audit_findings_transforms, + run_all_corrective_action_plan_transforms, + run_all_secondary_auditors_transforms, ) diff --git a/backend/audit/intakelib/transforms/runners.py b/backend/audit/intakelib/transforms/runners.py index 1841bcd98a..c69983a420 100644 --- a/backend/audit/intakelib/transforms/runners.py +++ b/backend/audit/intakelib/transforms/runners.py @@ -1,11 +1,30 @@ import logging from copy import deepcopy -from .xform_no_op import no_op - +from .xform_all_amount_expended_need_to_be_integers import ( + convert_amount_expended_to_integers, +) +from .xform_all_cluster_total_need_to_be_integers import ( + convert_cluster_total_to_integers, +) +from .xform_all_federal_program_total_need_to_be_integers import ( + convert_federal_program_total_to_integers, +) +from .xform_total_amount_expended_need_to_be_integers import ( + convert_total_amount_expended_to_integers, +) +from .xform_subrecipient_amount_need_to_be_integers import ( + convert_subrecipient_amount_to_integers, +) from .xform_insert_sequence_nums_into_notes_to_sefa import ( insert_sequence_nums_into_notes_to_sefa, ) +from .xform_number_of_findings_need_to_be_integers import ( + convert_number_of_findings_to_integers, +) +from .xform_loan_balance_need_to_be_integers import ( + convert_loan_balance_to_integers_or_na, +) # from .xform_filter_seq_numbers_where_there_are_no_values import filter_seq_numbers_where_there_are_no_values # from .xform_make_sure_notes_to_sefa_are_just_strings import make_sure_notes_to_sefa_are_just_strings @@ -13,15 +32,13 @@ trim_null_from_content_fields_in_notes_to_sefa, ) -from .xform_eins_need_to_be_strings import eins_need_to_be_strings +from .xform_all_fields_to_stripped_string import convert_to_stripped_string + from .xform_rename_additional_notes_sheet import ( rename_additional_notes_sheet_to_form_sheet, ) -from .xform_all_alns_need_to_be_strings import all_alns_need_to_be_strings -from .xform_all_passthrough_id_need_to_be_strings import ( - all_passthrough_id_need_to_be_strings, -) +from .xform_uniform_cluster_names import regenerate_uniform_cluster_names from .xform_reformat_prior_references import reformat_prior_references logger = logging.getLogger(__name__) @@ -39,18 +56,36 @@ def run_all_notes_to_sefa_transforms(ir): def run_all_additional_eins_transforms(ir): - return run_all_transforms(ir, additional_eins_transforms) + return run_all_transforms(ir, general_transforms) def run_all_federal_awards_transforms(ir): return run_all_transforms(ir, federal_awards_transforms) +def run_all_additional_ueis_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +def run_all_audit_findings_text_transforms(ir): + return run_all_transforms(ir, general_transforms) + + def run_all_audit_findings_transforms(ir): - return run_all_transforms(ir, audit_findings_transforms) + return run_all_transforms(ir, general_transforms) + + +def run_all_corrective_action_plan_transforms(ir): + return run_all_transforms(ir, general_transforms) -general_transforms = [no_op] +def run_all_secondary_auditors_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +general_transforms = [ + convert_to_stripped_string, +] notes_to_sefa_transforms = general_transforms + [ trim_null_from_content_fields_in_notes_to_sefa, @@ -58,13 +93,16 @@ def run_all_audit_findings_transforms(ir): insert_sequence_nums_into_notes_to_sefa, ] -additional_eins_transforms = general_transforms + [ - eins_need_to_be_strings, -] - federal_awards_transforms = general_transforms + [ - all_alns_need_to_be_strings, - all_passthrough_id_need_to_be_strings, + convert_amount_expended_to_integers, + convert_cluster_total_to_integers, + convert_federal_program_total_to_integers, + convert_total_amount_expended_to_integers, + convert_subrecipient_amount_to_integers, + convert_total_amount_expended_to_integers, + convert_number_of_findings_to_integers, + convert_loan_balance_to_integers_or_na, + regenerate_uniform_cluster_names, ] audit_findings_transforms = general_transforms + [ diff --git a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py deleted file mode 100644 index a60b907e60..0000000000 --- a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def all_alns_need_to_be_strings(ir): - agencies = get_range_by_name(ir, "federal_agency_prefix") - new_values = list(map(lambda v: str(v), agencies["values"])) - new_ir = replace_range_by_name(ir, "federal_agency_prefix", new_values) - - extensions = get_range_by_name(ir, "three_digit_extension") - new_values = list(map(lambda v: str(v), extensions["values"])) - new_ir = replace_range_by_name(ir, "three_digit_extension", new_values) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py new file mode 100644 index 0000000000..e7580d0772 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all amount expended to integers +def convert_amount_expended_to_integers(ir): + xform_ir = safe_int_conversion(ir, "amount_expended") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py new file mode 100644 index 0000000000..f598dc01e7 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py @@ -0,0 +1,12 @@ +import logging + +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all cluster totals to integers +def convert_cluster_total_to_integers(ir): + xform_ir = safe_int_conversion(ir, "cluster_total") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py new file mode 100644 index 0000000000..b15831fb1d --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all federal program totals to integers +def convert_federal_program_total_to_integers(ir): + xform_ir = safe_int_conversion(ir, "federal_program_total") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py new file mode 100644 index 0000000000..0d48b503f8 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py @@ -0,0 +1,23 @@ +import logging +from copy import deepcopy +from audit.intakelib.intermediate_representation import ( + replace_range_by_name, +) + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Strip all text fields of leading and trailing whitespace +def convert_to_stripped_string(ir): + new_ir = deepcopy(ir) + for sheet in ir: + # To ensure backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1, we check for both "AdditionalNotes" and "Form" + if sheet["name"] in {"AdditionalNotes", "Form", "Coversheet"}: + for range in sheet["ranges"]: + formatted_values = [ + None if v is None or (not str(v).strip()) else str(v).strip() + for v in range["values"] + ] + new_ir = replace_range_by_name(new_ir, range["name"], formatted_values) + return new_ir diff --git a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py deleted file mode 100644 index 165bdf3667..0000000000 --- a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def all_passthrough_id_need_to_be_strings(ir): - passthrough_ids = get_range_by_name(ir, "passthrough_identifying_number") - new_values = [str(v) if v is not None else None for v in passthrough_ids["values"]] - new_ir = replace_range_by_name(ir, "passthrough_identifying_number", new_values) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py deleted file mode 100644 index affeaa4d43..0000000000 --- a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def eins_need_to_be_strings(ir): - eins = get_range_by_name(ir, "additional_ein") - new_values = list(map(lambda v: str(v), eins["values"])) - new_ir = replace_range_by_name(ir, "additional_ein", new_values) - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py new file mode 100644 index 0000000000..6e8870ae2e --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert end of period loan balance to integers when applicable +def convert_loan_balance_to_integers_or_na(ir): + xform_ir = safe_int_conversion(ir, "loan_balance_at_audit_period_end", {"N/A"}) + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py b/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py deleted file mode 100644 index 6cf05eb0c7..0000000000 --- a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def make_sure_notes_to_sefa_are_just_strings(ir): - note_content = get_range_by_name(ir, "note_content") - note_title = get_range_by_name(ir, "note_title") - new_contents = [] - new_titles = [] - for indx, (note_con, note_tit) in enumerate( - zip(note_content["values"], note_title["values"]) - ): - new_contents.append(str(note_con).strip()) - new_titles.append(str(note_tit).strip()) - - new_ir = replace_range_by_name(ir, "note_content", new_contents) - new_ir = replace_range_by_name(new_ir, "note_title", new_titles) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py new file mode 100644 index 0000000000..427a522d17 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all number_of_audit_findings to integers +def convert_number_of_findings_to_integers(ir): + xform_ir = safe_int_conversion(ir, "number_of_audit_findings") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py index c9e997eb3c..0397c490fb 100644 --- a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py +++ b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py @@ -4,7 +4,8 @@ logger = logging.getLogger(__name__) -# This transform is needed for backwards compatibility with workbook templates 1.0.0 and 1.0.1 +# This transform is needed for backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1 +# Once we deprecate those versions, we can remove this transform def rename_additional_notes_sheet_to_form_sheet(ir): new_ir = deepcopy(ir) diff --git a/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py new file mode 100644 index 0000000000..c953ecfb8a --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all subrecipient_amount to integers +def convert_subrecipient_amount_to_integers(ir): + xform_ir = safe_int_conversion(ir, "subrecipient_amount") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py new file mode 100644 index 0000000000..b3b16d6cb2 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py @@ -0,0 +1,12 @@ +import logging + +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# convert total amount expended to integers +def convert_total_amount_expended_to_integers(ir): + xform_ir = safe_int_conversion(ir, "total_amount_expended") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py new file mode 100644 index 0000000000..f782dc132e --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py @@ -0,0 +1,27 @@ +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, + replace_range_by_name, +) + +logger = logging.getLogger(__name__) + + +def regenerate_uniform_cluster_names(ir): + state_cluster_names = get_range_by_name(ir, "state_cluster_name") + other_cluster_names = get_range_by_name(ir, "other_cluster_name") + uniform_state_cluster_name_values = [ + None if v is None or not str(v).strip() else str(v).strip().upper() + for v in state_cluster_names["values"] + ] + uniform_other_cluster_name_values = [ + None if v is None or not str(v).strip() else str(v).strip().upper() + for v in other_cluster_names["values"] + ] + new_ir = replace_range_by_name( + ir, "uniform_state_cluster_name", uniform_state_cluster_name_values + ) + xform_ir = replace_range_by_name( + new_ir, "uniform_other_cluster_name", uniform_other_cluster_name_values + ) + return xform_ir