diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx
new file mode 100644
index 0000000000..871257262a
Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx differ
diff --git a/backend/audit/intakelib/checks/check_additional_award_identification_present.py b/backend/audit/intakelib/checks/check_additional_award_identification_present.py
index 80c038faf6..2b72a4f785 100644
--- a/backend/audit/intakelib/checks/check_additional_award_identification_present.py
+++ b/backend/audit/intakelib/checks/check_additional_award_identification_present.py
@@ -8,7 +8,7 @@
REGEX_RD_EXTENSION,
REGEX_U_EXTENSION,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py
index ed580ba2d5..d594064edd 100644
--- a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py
+++ b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py
index 6ef76ed313..1bf908002e 100644
--- a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py
+++ b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py
@@ -4,7 +4,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py
index cef39db826..fdbedcb26b 100644
--- a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py
+++ b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py
@@ -2,7 +2,7 @@
from audit.intakelib.intermediate_representation import (
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_cluster_total.py b/backend/audit/intakelib/checks/check_cluster_total.py
index 53e3945f19..fadeb51fff 100644
--- a/backend/audit/intakelib/checks/check_cluster_total.py
+++ b/backend/audit/intakelib/checks/check_cluster_total.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_eins_are_not_empty.py b/backend/audit/intakelib/checks/check_eins_are_not_empty.py
index 2ad2ef6b35..6c0254aef2 100644
--- a/backend/audit/intakelib/checks/check_eins_are_not_empty.py
+++ b/backend/audit/intakelib/checks/check_eins_are_not_empty.py
@@ -1,6 +1,6 @@
from django.core.exceptions import ValidationError
import logging
-from .util import get_missing_value_errors
+from audit.intakelib.common import get_missing_value_errors
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py
similarity index 95%
rename from backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py
rename to backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py
index d28e043d53..449bea23fe 100644
--- a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py
+++ b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_federal_program_total.py b/backend/audit/intakelib/checks/check_federal_program_total.py
index d7389f418f..77e52e394f 100644
--- a/backend/audit/intakelib/checks/check_federal_program_total.py
+++ b/backend/audit/intakelib/checks/check_federal_program_total.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py
index afc8a12820..c9f631929f 100644
--- a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py
+++ b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py
@@ -4,7 +4,12 @@
from audit.intakelib.intermediate_representation import (
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple, appears_empty, is_value_na
+from audit.intakelib.common import (
+ get_message,
+ build_cell_error_tuple,
+ appears_empty,
+ is_value_marked_na,
+)
logger = logging.getLogger(__name__)
@@ -22,7 +27,7 @@ def prior_references_pattern(ir):
for index, prior_reference in enumerate(prior_references["values"]):
if (
not appears_empty(prior_reference)
- and (not is_value_na(prior_reference))
+ and (not is_value_marked_na(prior_reference))
and (not re.match(PRIOR_REFERENCES_REGEX, str(prior_reference)))
):
errors.append(
diff --git a/backend/audit/intakelib/checks/check_findings_grid_validation.py b/backend/audit/intakelib/checks/check_findings_grid_validation.py
index 7a910bddef..d22f0eebf6 100644
--- a/backend/audit/intakelib/checks/check_findings_grid_validation.py
+++ b/backend/audit/intakelib/checks/check_findings_grid_validation.py
@@ -2,7 +2,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
# Modified Opinion
# Other Matters
diff --git a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py
index 922b5d7997..e4ba189044 100644
--- a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py
+++ b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py
@@ -3,7 +3,7 @@
from audit.intakelib.intermediate_representation import (
extract_workbook_as_ir,
)
-from .util import get_names_of_all_ranges
+from audit.intakelib.common import get_names_of_all_ranges
from audit.fixtures.excel import FORM_SECTIONS
from audit.fixtures.excel import (
ADDITIONAL_UEIS_TEMPLATE,
diff --git a/backend/audit/intakelib/checks/check_is_right_workbook.py b/backend/audit/intakelib/checks/check_is_right_workbook.py
index 8ef87f12a3..81d7bc5a2e 100644
--- a/backend/audit/intakelib/checks/check_is_right_workbook.py
+++ b/backend/audit/intakelib/checks/check_is_right_workbook.py
@@ -4,7 +4,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_loan_balance_entries.py b/backend/audit/intakelib/checks/check_loan_balance_entries.py
index 96c33660eb..7b2be52750 100644
--- a/backend/audit/intakelib/checks/check_loan_balance_entries.py
+++ b/backend/audit/intakelib/checks/check_loan_balance_entries.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_loan_balance_present.py b/backend/audit/intakelib/checks/check_loan_balance_present.py
index f661f9b223..33e882dcf2 100644
--- a/backend/audit/intakelib/checks/check_loan_balance_present.py
+++ b/backend/audit/intakelib/checks/check_loan_balance_present.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_look_for_empty_rows.py b/backend/audit/intakelib/checks/check_look_for_empty_rows.py
index 437e0e2a9f..a7a805265e 100644
--- a/backend/audit/intakelib/checks/check_look_for_empty_rows.py
+++ b/backend/audit/intakelib/checks/check_look_for_empty_rows.py
@@ -2,7 +2,7 @@
import logging
from audit.intakelib.intermediate_representation import ranges_to_rows, appears_empty
-from .util import get_range_start_row
+from audit.intakelib.common import get_range_start_row
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_missing_required_fields.py b/backend/audit/intakelib/checks/check_missing_required_fields.py
index 3785bf3bf7..8f2d37a7cc 100644
--- a/backend/audit/intakelib/checks/check_missing_required_fields.py
+++ b/backend/audit/intakelib/checks/check_missing_required_fields.py
@@ -1,6 +1,6 @@
from django.core.exceptions import ValidationError
import logging
-from .util import get_missing_value_errors
+from audit.intakelib.common import get_missing_value_errors
from audit.fixtures.excel import FORM_SECTIONS
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_no_major_program_no_type.py b/backend/audit/intakelib/checks/check_no_major_program_no_type.py
index f6dbc639d3..a11dd02f5a 100644
--- a/backend/audit/intakelib/checks/check_no_major_program_no_type.py
+++ b/backend/audit/intakelib/checks/check_no_major_program_no_type.py
@@ -1,6 +1,6 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_no_repeat_findings.py b/backend/audit/intakelib/checks/check_no_repeat_findings.py
index efabc06e35..7f45a958f8 100644
--- a/backend/audit/intakelib/checks/check_no_repeat_findings.py
+++ b/backend/audit/intakelib/checks/check_no_repeat_findings.py
@@ -1,6 +1,10 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple, is_value_na
+from audit.intakelib.common import (
+ get_message,
+ build_cell_error_tuple,
+ is_value_marked_na,
+)
logger = logging.getLogger(__name__)
@@ -13,7 +17,7 @@ def no_repeat_findings(ir):
for ndx, (is_rep, prior) in enumerate(
zip(repeat_prior_reference["values"], prior_references["values"])
):
- if (is_rep == "N") and (not is_value_na(prior)):
+ if (is_rep == "N") and (not is_value_marked_na(prior)):
errors.append(
build_cell_error_tuple(
ir,
@@ -22,7 +26,7 @@ def no_repeat_findings(ir):
get_message("check_no_repeat_findings_when_n"),
)
)
- elif (is_rep == "Y") and ((not prior) or (is_value_na(prior))):
+ elif (is_rep == "Y") and ((not prior) or (is_value_marked_na(prior))):
errors.append(
build_cell_error_tuple(
ir,
diff --git a/backend/audit/intakelib/checks/check_other_cluster_names.py b/backend/audit/intakelib/checks/check_other_cluster_names.py
index 85a0ba6bd1..57de4dd0e6 100644
--- a/backend/audit/intakelib/checks/check_other_cluster_names.py
+++ b/backend/audit/intakelib/checks/check_other_cluster_names.py
@@ -1,6 +1,6 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py
index de11e8ab87..c63a9c68dd 100644
--- a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py
+++ b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py
@@ -1,6 +1,6 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_sequential_award_numbers.py b/backend/audit/intakelib/checks/check_sequential_award_numbers.py
index 06bae0bbd1..850f3f26c8 100644
--- a/backend/audit/intakelib/checks/check_sequential_award_numbers.py
+++ b/backend/audit/intakelib/checks/check_sequential_award_numbers.py
@@ -1,6 +1,6 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
import re
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_state_cluster_names.py b/backend/audit/intakelib/checks/check_state_cluster_names.py
index f3f692e5f2..8e984eebbc 100644
--- a/backend/audit/intakelib/checks/check_state_cluster_names.py
+++ b/backend/audit/intakelib/checks/check_state_cluster_names.py
@@ -1,6 +1,6 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_total_amount_expended.py b/backend/audit/intakelib/checks/check_total_amount_expended.py
index e9a041bed9..6e500a490f 100644
--- a/backend/audit/intakelib/checks/check_total_amount_expended.py
+++ b/backend/audit/intakelib/checks/check_total_amount_expended.py
@@ -3,7 +3,7 @@
get_range_values_by_name,
get_range_by_name,
)
-from .util import get_message, build_cell_error_tuple
+from audit.intakelib.common import get_message, build_cell_error_tuple
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_uei_exists.py b/backend/audit/intakelib/checks/check_uei_exists.py
index 0648d2679f..bc88ec839f 100644
--- a/backend/audit/intakelib/checks/check_uei_exists.py
+++ b/backend/audit/intakelib/checks/check_uei_exists.py
@@ -1,6 +1,10 @@
import logging
from audit.intakelib.intermediate_representation import get_range_by_name
-from .util import list_contains_non_null_values, get_message, build_range_error_tuple
+from audit.intakelib.common import (
+ list_contains_non_null_values,
+ get_message,
+ build_range_error_tuple,
+)
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/check_y_or_n__fields.py b/backend/audit/intakelib/checks/check_y_or_n__fields.py
index fe40db7d24..0af2674761 100644
--- a/backend/audit/intakelib/checks/check_y_or_n__fields.py
+++ b/backend/audit/intakelib/checks/check_y_or_n__fields.py
@@ -1,5 +1,5 @@
import logging
-from .util import invalid_y_or_n_entry
+from audit.intakelib.common import invalid_y_or_n_entry
from audit.fixtures.excel import FORM_SECTIONS
logger = logging.getLogger(__name__)
diff --git a/backend/audit/intakelib/checks/runners.py b/backend/audit/intakelib/checks/runners.py
index 22aa490255..34c71bd427 100644
--- a/backend/audit/intakelib/checks/runners.py
+++ b/backend/audit/intakelib/checks/runners.py
@@ -29,7 +29,7 @@
from .check_federal_program_total import federal_program_total_is_correct
from .check_cluster_total import cluster_total_is_correct
from .check_total_amount_expended import total_amount_expended_is_correct
-from .check_federal_award_passed_passed_through_optional import (
+from .check_federal_award_passed_through_optional import (
federal_award_amount_passed_through_optional,
)
from .check_cardinality_of_passthrough_names_and_ids import (
diff --git a/backend/audit/intakelib/common/__init__.py b/backend/audit/intakelib/common/__init__.py
new file mode 100644
index 0000000000..665ee1f767
--- /dev/null
+++ b/backend/audit/intakelib/common/__init__.py
@@ -0,0 +1,14 @@
+# flake8: noqa: F401
+from .util import (
+ get_message,
+ build_cell_error_tuple,
+ appears_empty,
+ is_value_marked_na,
+ get_names_of_all_ranges,
+ get_range_start_row,
+ get_missing_value_errors,
+ invalid_y_or_n_entry,
+ safe_int_conversion,
+ list_contains_non_null_values,
+ build_range_error_tuple,
+)
diff --git a/backend/audit/intakelib/checks/error_messages.py b/backend/audit/intakelib/common/error_messages.py
similarity index 98%
rename from backend/audit/intakelib/checks/error_messages.py
rename to backend/audit/intakelib/common/error_messages.py
index 5479c0bc2b..e968c6c6cb 100644
--- a/backend/audit/intakelib/checks/error_messages.py
+++ b/backend/audit/intakelib/common/error_messages.py
@@ -55,4 +55,5 @@
"check_federal_award_amount_passed_through_not_allowed": "When Federal Award Passed Through is N, Amount Passed Through must be empty",
"check_loan_balance": "The loan balance is currently set to {}. It should either be a positive number, N/A, or left empty",
"check_cardinality_of_passthrough_names_and_ids": "You used a | (bar character) to indicate multiple passthrough names and IDs; you must provide equal numbers of names and IDs. You provided {} name{} and {} ID{}",
+ "check_integer_values": "{} is not a valid integer",
}
diff --git a/backend/audit/intakelib/checks/util.py b/backend/audit/intakelib/common/util.py
similarity index 66%
rename from backend/audit/intakelib/checks/util.py
rename to backend/audit/intakelib/common/util.py
index 4aa9b3d322..42ebfba623 100644
--- a/backend/audit/intakelib/checks/util.py
+++ b/backend/audit/intakelib/common/util.py
@@ -2,7 +2,9 @@
import logging
from audit.intakelib.intermediate_representation import (
get_range_by_name,
+ replace_range_by_name,
)
+from django.core.exceptions import ValidationError
logger = logging.getLogger(__name__)
@@ -59,7 +61,7 @@ def appears_empty(v):
return (v is None) or (str(v).strip() == "")
-def is_value_na(v):
+def is_value_marked_na(v):
value = str(v).strip()
return value == "N/A"
@@ -113,3 +115,38 @@ def get_names_of_all_ranges(data):
if "name" in range_item:
names.append(range_item["name"])
return names
+
+
+def safe_int_conversion(ir, range_name, other_values_allowed=None):
+ range_data = get_range_by_name(ir, range_name)
+ errors = []
+ new_values = []
+ if range_data:
+ for index, value in enumerate(range_data["values"]):
+ try:
+ float_value = float(value)
+ if float_value.is_integer():
+ new_values.append(int(float_value))
+ else:
+ raise ValueError
+ except (ValueError, TypeError):
+ # If the value is None, we keep it. This is because some int fields are optional.
+ # For non optional fields, there is a check for missing required fields that will raise an error.
+ if (value is None) or (
+ other_values_allowed and value in other_values_allowed
+ ):
+ new_values.append(value)
+ else:
+ errors.append(
+ build_cell_error_tuple(
+ ir,
+ range_data,
+ index,
+ get_message("check_integer_values").format(value),
+ )
+ )
+ if len(errors) > 0:
+ logger.info("Raising a validation error.")
+ raise ValidationError(errors)
+ new_ir = replace_range_by_name(ir, range_name, new_values)
+ return new_ir
diff --git a/backend/audit/intakelib/mapping_additional_eins.py b/backend/audit/intakelib/mapping_additional_eins.py
index 2f306ab19e..31180723e4 100644
--- a/backend/audit/intakelib/mapping_additional_eins.py
+++ b/backend/audit/intakelib/mapping_additional_eins.py
@@ -46,8 +46,8 @@ def extract_additional_eins(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_EINS)
- run_all_additional_eins_checks(ir)
xform_ir = run_all_additional_eins_transforms(ir)
+ run_all_additional_eins_checks(xform_ir)
result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/mapping_additional_ueis.py b/backend/audit/intakelib/mapping_additional_ueis.py
index ec3c65b88d..d58696a862 100644
--- a/backend/audit/intakelib/mapping_additional_ueis.py
+++ b/backend/audit/intakelib/mapping_additional_ueis.py
@@ -22,7 +22,7 @@
)
from .mapping_meta import meta_mapping
-
+from .transforms import run_all_additional_ueis_transforms
from .checks import run_all_general_checks, run_all_additional_ueis_checks
logger = logging.getLogger(__name__)
@@ -43,8 +43,9 @@ def extract_additional_ueis(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_UEIS)
- run_all_additional_ueis_checks(ir)
- result = _extract_generic_data(ir, params)
+ xform_ir = run_all_additional_ueis_transforms(ir)
+ run_all_additional_ueis_checks(xform_ir)
+ result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/mapping_audit_findings.py b/backend/audit/intakelib/mapping_audit_findings.py
index a481605efa..6b29bd02b8 100644
--- a/backend/audit/intakelib/mapping_audit_findings.py
+++ b/backend/audit/intakelib/mapping_audit_findings.py
@@ -43,9 +43,9 @@ def extract_audit_findings(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE)
- new_ir = run_all_audit_findings_transforms(ir)
- run_all_audit_finding_checks(new_ir)
- result = _extract_generic_data(new_ir, params)
+ xform_ir = run_all_audit_findings_transforms(ir)
+ run_all_audit_finding_checks(xform_ir)
+ result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/mapping_audit_findings_text.py b/backend/audit/intakelib/mapping_audit_findings_text.py
index dcc0887fff..a7817e8e3b 100644
--- a/backend/audit/intakelib/mapping_audit_findings_text.py
+++ b/backend/audit/intakelib/mapping_audit_findings_text.py
@@ -21,7 +21,7 @@
)
from .mapping_meta import meta_mapping
-
+from .transforms import run_all_audit_findings_text_transforms
from .checks import run_all_general_checks, run_all_audit_findings_text_checks
logger = logging.getLogger(__name__)
@@ -42,8 +42,9 @@ def extract_audit_findings_text(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_TEXT)
- run_all_audit_findings_text_checks(ir)
- result = _extract_generic_data(ir, params)
+ xform_ir = run_all_audit_findings_text_transforms(ir)
+ run_all_audit_findings_text_checks(xform_ir)
+ result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/mapping_corrective_action_plan.py b/backend/audit/intakelib/mapping_corrective_action_plan.py
index 1068f784cb..82ae5b05ae 100644
--- a/backend/audit/intakelib/mapping_corrective_action_plan.py
+++ b/backend/audit/intakelib/mapping_corrective_action_plan.py
@@ -22,6 +22,7 @@
from .mapping_meta import meta_mapping
from .checks import run_all_general_checks, run_all_corrective_action_plan_checks
+from .transforms import run_all_corrective_action_plan_transforms
logger = logging.getLogger(__name__)
@@ -41,8 +42,9 @@ def extract_corrective_action_plan(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.CORRECTIVE_ACTION_PLAN)
- run_all_corrective_action_plan_checks(ir)
- result = _extract_generic_data(ir, params)
+ xform_ir = run_all_corrective_action_plan_transforms(ir)
+ run_all_corrective_action_plan_checks(xform_ir)
+ result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/mapping_secondary_auditors.py b/backend/audit/intakelib/mapping_secondary_auditors.py
index 02a3c31d73..acd99a94e3 100644
--- a/backend/audit/intakelib/mapping_secondary_auditors.py
+++ b/backend/audit/intakelib/mapping_secondary_auditors.py
@@ -18,7 +18,7 @@
from .mapping_meta import meta_mapping
from .checks import run_all_general_checks, run_all_secondary_auditors_checks
-
+from .transforms import run_all_secondary_auditors_transforms
from .intermediate_representation import (
extract_workbook_as_ir,
_extract_generic_data,
@@ -42,8 +42,9 @@ def extract_secondary_auditors(file):
ir = extract_workbook_as_ir(file)
run_all_general_checks(ir, FORM_SECTIONS.SECONDARY_AUDITORS)
- run_all_secondary_auditors_checks(ir)
- result = _extract_generic_data(ir, params)
+ xform_ir = run_all_secondary_auditors_transforms(ir)
+ run_all_secondary_auditors_checks(xform_ir)
+ result = _extract_generic_data(xform_ir, params)
return result
diff --git a/backend/audit/intakelib/transforms/__init__.py b/backend/audit/intakelib/transforms/__init__.py
index e42e5a6f12..b59f5eb09f 100644
--- a/backend/audit/intakelib/transforms/__init__.py
+++ b/backend/audit/intakelib/transforms/__init__.py
@@ -3,5 +3,9 @@
run_all_notes_to_sefa_transforms,
run_all_additional_eins_transforms,
run_all_federal_awards_transforms,
+ run_all_additional_ueis_transforms,
+ run_all_audit_findings_text_transforms,
run_all_audit_findings_transforms,
+ run_all_corrective_action_plan_transforms,
+ run_all_secondary_auditors_transforms,
)
diff --git a/backend/audit/intakelib/transforms/runners.py b/backend/audit/intakelib/transforms/runners.py
index 1841bcd98a..c69983a420 100644
--- a/backend/audit/intakelib/transforms/runners.py
+++ b/backend/audit/intakelib/transforms/runners.py
@@ -1,11 +1,30 @@
import logging
from copy import deepcopy
-from .xform_no_op import no_op
-
+from .xform_all_amount_expended_need_to_be_integers import (
+ convert_amount_expended_to_integers,
+)
+from .xform_all_cluster_total_need_to_be_integers import (
+ convert_cluster_total_to_integers,
+)
+from .xform_all_federal_program_total_need_to_be_integers import (
+ convert_federal_program_total_to_integers,
+)
+from .xform_total_amount_expended_need_to_be_integers import (
+ convert_total_amount_expended_to_integers,
+)
+from .xform_subrecipient_amount_need_to_be_integers import (
+ convert_subrecipient_amount_to_integers,
+)
from .xform_insert_sequence_nums_into_notes_to_sefa import (
insert_sequence_nums_into_notes_to_sefa,
)
+from .xform_number_of_findings_need_to_be_integers import (
+ convert_number_of_findings_to_integers,
+)
+from .xform_loan_balance_need_to_be_integers import (
+ convert_loan_balance_to_integers_or_na,
+)
# from .xform_filter_seq_numbers_where_there_are_no_values import filter_seq_numbers_where_there_are_no_values
# from .xform_make_sure_notes_to_sefa_are_just_strings import make_sure_notes_to_sefa_are_just_strings
@@ -13,15 +32,13 @@
trim_null_from_content_fields_in_notes_to_sefa,
)
-from .xform_eins_need_to_be_strings import eins_need_to_be_strings
+from .xform_all_fields_to_stripped_string import convert_to_stripped_string
+
from .xform_rename_additional_notes_sheet import (
rename_additional_notes_sheet_to_form_sheet,
)
-from .xform_all_alns_need_to_be_strings import all_alns_need_to_be_strings
-from .xform_all_passthrough_id_need_to_be_strings import (
- all_passthrough_id_need_to_be_strings,
-)
+from .xform_uniform_cluster_names import regenerate_uniform_cluster_names
from .xform_reformat_prior_references import reformat_prior_references
logger = logging.getLogger(__name__)
@@ -39,18 +56,36 @@ def run_all_notes_to_sefa_transforms(ir):
def run_all_additional_eins_transforms(ir):
- return run_all_transforms(ir, additional_eins_transforms)
+ return run_all_transforms(ir, general_transforms)
def run_all_federal_awards_transforms(ir):
return run_all_transforms(ir, federal_awards_transforms)
+def run_all_additional_ueis_transforms(ir):
+ return run_all_transforms(ir, general_transforms)
+
+
+def run_all_audit_findings_text_transforms(ir):
+ return run_all_transforms(ir, general_transforms)
+
+
def run_all_audit_findings_transforms(ir):
- return run_all_transforms(ir, audit_findings_transforms)
+ return run_all_transforms(ir, general_transforms)
+
+
+def run_all_corrective_action_plan_transforms(ir):
+ return run_all_transforms(ir, general_transforms)
-general_transforms = [no_op]
+def run_all_secondary_auditors_transforms(ir):
+ return run_all_transforms(ir, general_transforms)
+
+
+general_transforms = [
+ convert_to_stripped_string,
+]
notes_to_sefa_transforms = general_transforms + [
trim_null_from_content_fields_in_notes_to_sefa,
@@ -58,13 +93,16 @@ def run_all_audit_findings_transforms(ir):
insert_sequence_nums_into_notes_to_sefa,
]
-additional_eins_transforms = general_transforms + [
- eins_need_to_be_strings,
-]
-
federal_awards_transforms = general_transforms + [
- all_alns_need_to_be_strings,
- all_passthrough_id_need_to_be_strings,
+ convert_amount_expended_to_integers,
+ convert_cluster_total_to_integers,
+ convert_federal_program_total_to_integers,
+ convert_total_amount_expended_to_integers,
+ convert_subrecipient_amount_to_integers,
+ convert_total_amount_expended_to_integers,
+ convert_number_of_findings_to_integers,
+ convert_loan_balance_to_integers_or_na,
+ regenerate_uniform_cluster_names,
]
audit_findings_transforms = general_transforms + [
diff --git a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py
deleted file mode 100644
index a60b907e60..0000000000
--- a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import logging
-from audit.intakelib.intermediate_representation import (
- get_range_by_name,
- replace_range_by_name,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def all_alns_need_to_be_strings(ir):
- agencies = get_range_by_name(ir, "federal_agency_prefix")
- new_values = list(map(lambda v: str(v), agencies["values"]))
- new_ir = replace_range_by_name(ir, "federal_agency_prefix", new_values)
-
- extensions = get_range_by_name(ir, "three_digit_extension")
- new_values = list(map(lambda v: str(v), extensions["values"]))
- new_ir = replace_range_by_name(ir, "three_digit_extension", new_values)
-
- return new_ir
diff --git a/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py
new file mode 100644
index 0000000000..e7580d0772
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py
@@ -0,0 +1,11 @@
+import logging
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert all amount expended to integers
+def convert_amount_expended_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "amount_expended")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py
new file mode 100644
index 0000000000..f598dc01e7
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py
@@ -0,0 +1,12 @@
+import logging
+
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert all cluster totals to integers
+def convert_cluster_total_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "cluster_total")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py
new file mode 100644
index 0000000000..b15831fb1d
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py
@@ -0,0 +1,11 @@
+import logging
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert all federal program totals to integers
+def convert_federal_program_total_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "federal_program_total")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py
new file mode 100644
index 0000000000..0d48b503f8
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py
@@ -0,0 +1,23 @@
+import logging
+from copy import deepcopy
+from audit.intakelib.intermediate_representation import (
+ replace_range_by_name,
+)
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Strip all text fields of leading and trailing whitespace
+def convert_to_stripped_string(ir):
+ new_ir = deepcopy(ir)
+ for sheet in ir:
+ # To ensure backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1, we check for both "AdditionalNotes" and "Form"
+ if sheet["name"] in {"AdditionalNotes", "Form", "Coversheet"}:
+ for range in sheet["ranges"]:
+ formatted_values = [
+ None if v is None or (not str(v).strip()) else str(v).strip()
+ for v in range["values"]
+ ]
+ new_ir = replace_range_by_name(new_ir, range["name"], formatted_values)
+ return new_ir
diff --git a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py
deleted file mode 100644
index 165bdf3667..0000000000
--- a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py
+++ /dev/null
@@ -1,15 +0,0 @@
-import logging
-from audit.intakelib.intermediate_representation import (
- get_range_by_name,
- replace_range_by_name,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def all_passthrough_id_need_to_be_strings(ir):
- passthrough_ids = get_range_by_name(ir, "passthrough_identifying_number")
- new_values = [str(v) if v is not None else None for v in passthrough_ids["values"]]
- new_ir = replace_range_by_name(ir, "passthrough_identifying_number", new_values)
-
- return new_ir
diff --git a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py
deleted file mode 100644
index affeaa4d43..0000000000
--- a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import logging
-from audit.intakelib.intermediate_representation import (
- get_range_by_name,
- replace_range_by_name,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def eins_need_to_be_strings(ir):
- eins = get_range_by_name(ir, "additional_ein")
- new_values = list(map(lambda v: str(v), eins["values"]))
- new_ir = replace_range_by_name(ir, "additional_ein", new_values)
- return new_ir
diff --git a/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py
new file mode 100644
index 0000000000..6e8870ae2e
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py
@@ -0,0 +1,11 @@
+import logging
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert end of period loan balance to integers when applicable
+def convert_loan_balance_to_integers_or_na(ir):
+ xform_ir = safe_int_conversion(ir, "loan_balance_at_audit_period_end", {"N/A"})
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py b/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py
deleted file mode 100644
index 6cf05eb0c7..0000000000
--- a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py
+++ /dev/null
@@ -1,24 +0,0 @@
-import logging
-from audit.intakelib.intermediate_representation import (
- get_range_by_name,
- replace_range_by_name,
-)
-
-logger = logging.getLogger(__name__)
-
-
-def make_sure_notes_to_sefa_are_just_strings(ir):
- note_content = get_range_by_name(ir, "note_content")
- note_title = get_range_by_name(ir, "note_title")
- new_contents = []
- new_titles = []
- for indx, (note_con, note_tit) in enumerate(
- zip(note_content["values"], note_title["values"])
- ):
- new_contents.append(str(note_con).strip())
- new_titles.append(str(note_tit).strip())
-
- new_ir = replace_range_by_name(ir, "note_content", new_contents)
- new_ir = replace_range_by_name(new_ir, "note_title", new_titles)
-
- return new_ir
diff --git a/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py
new file mode 100644
index 0000000000..427a522d17
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py
@@ -0,0 +1,11 @@
+import logging
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert all number_of_audit_findings to integers
+def convert_number_of_findings_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "number_of_audit_findings")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py
index c9e997eb3c..0397c490fb 100644
--- a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py
+++ b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py
@@ -4,7 +4,8 @@
logger = logging.getLogger(__name__)
-# This transform is needed for backwards compatibility with workbook templates 1.0.0 and 1.0.1
+# This transform is needed for backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1
+# Once we deprecate those versions, we can remove this transform
def rename_additional_notes_sheet_to_form_sheet(ir):
new_ir = deepcopy(ir)
diff --git a/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py
new file mode 100644
index 0000000000..c953ecfb8a
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py
@@ -0,0 +1,11 @@
+import logging
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# Convert all subrecipient_amount to integers
+def convert_subrecipient_amount_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "subrecipient_amount")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py
new file mode 100644
index 0000000000..b3b16d6cb2
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py
@@ -0,0 +1,12 @@
+import logging
+
+from audit.intakelib.common import safe_int_conversion
+
+logger = logging.getLogger(__name__)
+
+
+# DESCRIPTION
+# convert total amount expended to integers
+def convert_total_amount_expended_to_integers(ir):
+ xform_ir = safe_int_conversion(ir, "total_amount_expended")
+ return xform_ir
diff --git a/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py
new file mode 100644
index 0000000000..f782dc132e
--- /dev/null
+++ b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py
@@ -0,0 +1,27 @@
+import logging
+from audit.intakelib.intermediate_representation import (
+ get_range_by_name,
+ replace_range_by_name,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def regenerate_uniform_cluster_names(ir):
+ state_cluster_names = get_range_by_name(ir, "state_cluster_name")
+ other_cluster_names = get_range_by_name(ir, "other_cluster_name")
+ uniform_state_cluster_name_values = [
+ None if v is None or not str(v).strip() else str(v).strip().upper()
+ for v in state_cluster_names["values"]
+ ]
+ uniform_other_cluster_name_values = [
+ None if v is None or not str(v).strip() else str(v).strip().upper()
+ for v in other_cluster_names["values"]
+ ]
+ new_ir = replace_range_by_name(
+ ir, "uniform_state_cluster_name", uniform_state_cluster_name_values
+ )
+ xform_ir = replace_range_by_name(
+ new_ir, "uniform_other_cluster_name", uniform_other_cluster_name_values
+ )
+ return xform_ir