diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-aln-prefixes.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-aln-prefixes.xlsx new file mode 100644 index 0000000000..19223f11d4 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-aln-prefixes.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/aln_read_as_decimal_values/has-aln-prefix-returned-as-decimal-numbers.xlsx b/backend/audit/fixtures/workbooks/should_pass/aln_read_as_decimal_values/has-aln-prefix-returned-as-decimal-numbers.xlsx new file mode 100644 index 0000000000..bc8c0e6c6f Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_pass/aln_read_as_decimal_values/has-aln-prefix-returned-as-decimal-numbers.xlsx differ diff --git a/backend/audit/intakelib/checks/check_aln_prefix_pattern.py b/backend/audit/intakelib/checks/check_aln_prefix_pattern.py new file mode 100644 index 0000000000..18d63120ed --- /dev/null +++ b/backend/audit/intakelib/checks/check_aln_prefix_pattern.py @@ -0,0 +1,33 @@ +import logging +import re +from django.conf import settings + +from django.core.exceptions import ValidationError +from ..intermediate_representation import ( + get_range_by_name, +) +from ..common import get_message, build_cell_error_tuple + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# The ALN prefix represents a two-digit number ## +# TESTED BY has_bad_aln_prefix.xlsx +def aln_agency_prefix(ir): + prefixes = get_range_by_name(ir, "federal_agency_prefix") + errors = [] + for index, prefix in enumerate(prefixes["values"]): + if not re.match(settings.REGEX_ALN_PREFIX, str(prefix)): + errors.append( + build_cell_error_tuple( + ir, + prefixes, + index, + get_message("check_aln_prefix_invalid"), + ) + ) + + if len(errors) > 0: + logger.info("Raising a validation error.") + raise ValidationError(errors) diff --git a/backend/audit/intakelib/checks/check_finding_award_references_pattern.py b/backend/audit/intakelib/checks/check_finding_award_references_pattern.py index e265211253..838d54e7c4 100644 --- a/backend/audit/intakelib/checks/check_finding_award_references_pattern.py +++ b/backend/audit/intakelib/checks/check_finding_award_references_pattern.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -# A version of these regexes also exists in Base.libsonnet +# A version of this regex also exists in Base.libsonnet AWARD_REFERENCES_REGEX = r"^AWARD-(?!0000)[0-9]{4}$" diff --git a/backend/audit/intakelib/checks/runners.py b/backend/audit/intakelib/checks/runners.py index 4ef792a63a..f3a5e2f487 100644 --- a/backend/audit/intakelib/checks/runners.py +++ b/backend/audit/intakelib/checks/runners.py @@ -50,6 +50,7 @@ from .check_no_repeat_findings import no_repeat_findings from .check_findings_grid_validation import findings_grid_validation from .check_finding_prior_references_pattern import prior_references_pattern +from .check_aln_prefix_pattern import aln_agency_prefix logger = logging.getLogger(__name__) @@ -75,6 +76,7 @@ no_major_program_no_type, all_unique_award_numbers, sequential_award_numbers, + aln_agency_prefix, aln_three_digit_extension, additional_award_identification, federal_program_total_is_correct, diff --git a/backend/audit/intakelib/common/error_messages.py b/backend/audit/intakelib/common/error_messages.py index f2bd641dd7..e5d0e2b33c 100644 --- a/backend/audit/intakelib/common/error_messages.py +++ b/backend/audit/intakelib/common/error_messages.py @@ -66,6 +66,7 @@ "check_aln_three_digit_extension_invalid": "The three digit extension should follow one of these formats: ###, RD#, or U##, where # represents a number", "check_prior_references_invalid": "Prior references must be N/A or a comma-separated list of values in the format 20##-###, for example, 2019-001, 2019-002", "check_award_references_invalid": "Award references must be of the form AWARD-####, for example, AWARD-0001", + "check_aln_prefix_invalid": "The federal agency prefix should be a two-digit value, for example, 10", "check_additional_award_identification_present": "Missing additional award identification", "check_federal_program_total": "Federal program total is {}, but should be {}", "check_cluster_total": "This cluster total is {}, but should be {}", diff --git a/backend/audit/intakelib/transforms/runners.py b/backend/audit/intakelib/transforms/runners.py index e68d333a96..bff49412bb 100644 --- a/backend/audit/intakelib/transforms/runners.py +++ b/backend/audit/intakelib/transforms/runners.py @@ -42,6 +42,7 @@ from .xform_uniform_cluster_names import regenerate_uniform_cluster_names from .xform_reformat_prior_references import reformat_prior_references from .xform_reformat_award_references import reformat_award_reference +from .xform_reformat_agency_prefix import reformat_federal_agency_prefix logger = logging.getLogger(__name__) @@ -105,6 +106,7 @@ def run_all_secondary_auditors_transforms(ir): convert_number_of_findings_to_integers, convert_loan_balance_to_integers_or_na, regenerate_uniform_cluster_names, + reformat_federal_agency_prefix, generate_cfda_keys, ] diff --git a/backend/audit/intakelib/transforms/xform_reformat_agency_prefix.py b/backend/audit/intakelib/transforms/xform_reformat_agency_prefix.py new file mode 100644 index 0000000000..b0fc349285 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_reformat_agency_prefix.py @@ -0,0 +1,15 @@ +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, + replace_range_by_name, +) + +logger = logging.getLogger(__name__) + + +# Tested by has-aln-prefix-entered-as-numbers.xlsx +def reformat_federal_agency_prefix(ir): + references = get_range_by_name(ir, "federal_agency_prefix") + new_values = list(map(lambda v: v.split(".")[0] if v else v, references["values"])) + new_ir = replace_range_by_name(ir, "federal_agency_prefix", new_values) + return new_ir diff --git a/backend/audit/intakelib/transforms/xform_reformat_award_references.py b/backend/audit/intakelib/transforms/xform_reformat_award_references.py index b9b7b4a683..9275fab191 100644 --- a/backend/audit/intakelib/transforms/xform_reformat_award_references.py +++ b/backend/audit/intakelib/transforms/xform_reformat_award_references.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -# Tested by has_lowercase_award_reference.py +# Tested by has_lowercase_award_reference.xlsx def reformat_award_reference(ir): references = get_range_by_name(ir, "award_reference") new_values = list(map(lambda v: v.upper() if v else v, references["values"])) diff --git a/backend/schemas/source/base/Base.libsonnet b/backend/schemas/source/base/Base.libsonnet index e713ac060e..eb7709ba34 100644 --- a/backend/schemas/source/base/Base.libsonnet +++ b/backend/schemas/source/base/Base.libsonnet @@ -67,8 +67,8 @@ local Meta = { }, }; -local REGEX_ALN_PREFIX = '^([0-9]{2})$'; # A python version of these regexes also exists in settings.py +local REGEX_ALN_PREFIX = '^([0-9]{2})$'; local REGEX_RD_EXTENSION = 'RD[0-9]?'; local REGEX_THREE_DIGIT_EXTENSION = '[0-9]{3}[A-Za-z]{0,1}'; local REGEX_U_EXTENSION = 'U[0-9]{2}'; diff --git a/backend/static/js/globals.js b/backend/static/js/globals.js index 11eff355f0..401701c42d 100644 --- a/backend/static/js/globals.js +++ b/backend/static/js/globals.js @@ -1,4 +1,4 @@ -export const UPLOAD_TIMEOUT = 30000; // 30s +export const UPLOAD_TIMEOUT = 60000; // 60s export const UPLOAD_URLS = { 'federal-awards': 'federal-awards-expended',