#2282 Fixed schema validation bug for federal awards section (#2362)

* #2282 Fixed schema validation bug for federal awards section * #2282 Updated FederalAward schema to allow negative numbers and applied jsonnet formatting
GSA-TTS · Oct 3, 2023 · 97b63c5 · 97b63c5
1 parent 5c75dd4
commit 97b63c5
Show file tree

Hide file tree

Showing 20 changed files with 169 additions and 124 deletions.
diff --git a/backend/audit/excel.py b/backend/audit/excel.py
@@ -508,16 +508,54 @@ def _extract_column_data(workbook, result, params):
                 set_fn(result, f"{parent_target}", entries)
 
 
+def _has_only_one_field_with_value_0(my_dict, field_name):
+    """Check if the dictionary has exactly one field with the provided name and its value is 0"""
+    return len(my_dict) == 1 and my_dict.get(field_name) == 0
+
+
 def _remove_empty_award_entries(data):
     """Removes empty award entries from the data"""
-    indexed_awards = []
+    awards = []
     for award in data.get("FederalAwards", {}).get("federal_awards", []):
-        if "program" in award:
-            program = award["program"]
-            if FEDERAL_AGENCY_PREFIX in program:
-                indexed_awards.append(award)
+        if not all(
+            [
+                "direct_or_indirect_award" not in award,
+                "loan_or_loan_guarantee" not in award,
+                "subrecipients" not in award,
+                "program" in award
+                and _has_only_one_field_with_value_0(
+                    award["program"], "federal_program_total"
+                ),
+                "cluster" in award
+                and _has_only_one_field_with_value_0(award["cluster"], "cluster_total"),
+            ]
+        ):
+            awards.append(award)
     if "FederalAwards" in data:
         # Update the federal_awards with the valid awards
+        data["FederalAwards"]["federal_awards"] = awards
+
+    return data
+
+
+def _add_required_fields(data):
+    """Adds empty parent fields to the json object to allow for proper schema validation / indexing"""
+    indexed_awards = []
+    for award in data.get("FederalAwards", {}).get("federal_awards", []):
+        if "cluster" not in award:
+            award["cluster"] = {}
+        if "direct_or_indirect_award" not in award:
+            award["direct_or_indirect_award"] = {}
+        if "loan_or_loan_guarantee" not in award:
+            award["loan_or_loan_guarantee"] = {}
+        if "program" not in award:
+            award["program"] = {}
+        if "subrecipients" not in award:
+            award["subrecipients"] = {}
+        indexed_awards.append(award)
+
+    if "FederalAwards" in data:
+        # Update the federal_awards with all required fields
         data["FederalAwards"]["federal_awards"] = indexed_awards
 
     return data
@@ -536,7 +574,9 @@ def extract_federal_awards(file):
         template["title_row"],
     )
     result = _extract_data(file, params)
-    return _remove_empty_award_entries(result)
+    result = _remove_empty_award_entries(result)
+    result = _add_required_fields(result)
+    return result
 
 
 def extract_corrective_action_plan(file):

diff --git a/backend/audit/validators.py b/backend/audit/validators.py
@@ -3,7 +3,6 @@
 import logging
 from jsonschema import Draft7Validator, FormatChecker, validate
 from jsonschema.exceptions import ValidationError as JSONSchemaValidationError
-from jsonschema.exceptions import SchemaError as JSONSchemaError
 
 from django.core.exceptions import ValidationError
 
@@ -209,18 +208,8 @@ def validate_federal_award_json(value):
     schema_path = settings.SECTION_SCHEMA_DIR / "FederalAwards.schema.json"
     schema = json.loads(schema_path.read_text(encoding="utf-8"))
 
-    # FIXME: For some classes of error, this approach
-    # will go into an infinite/recursive loop in the iterator.
-    # There needs to either be a way to solve that, or we need
-    # to not use this approach.
-    # validator = Draft7Validator(schema)
-    # errors = list(validator.iter_errors(value))
-    # The side-effect is that now I only do one error at a time...
-    errors = []
-    try:
-        validate(schema=schema, instance=value)
-    except (JSONSchemaValidationError, JSONSchemaError) as e:
-        errors = [e]
+    validator = Draft7Validator(schema)
+    errors = list(validator.iter_errors(value))
     if len(errors) > 0:
         raise ValidationError(message=_federal_awards_json_error(errors))
 

diff --git a/backend/schemas/output/excel/json/federal-awards-workbook.json b/backend/schemas/output/excel/json/federal-awards-workbook.json
@@ -89,8 +89,8 @@
                "format": "dollar",
                "formula": "=SUM('Form'!F$FIRSTROW:F$LASTROW)",
                "help": {
-                  "link": "https://fac.gov/documentation/validation/#positive_number",
-                  "text": "The number must be zero or greater"
+                  "link": "https://fac.gov/documentation/validation/#any_number",
+                  "text": "Must be a number"
                },
                "keep_locked": true,
                "range_cell": "B5",
@@ -99,9 +99,9 @@
                "title_cell": "A5",
                "type": "single_cell",
                "validation": {
-                  "custom_error": "This cell must be a positive number",
-                  "custom_title": "Positive numbers",
-                  "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
+                  "custom_error": "This cell must be a number",
+                  "custom_title": "Numbers",
+                  "formula1": "=ISNUMBER(FIRSTCELLREF)",
                   "type": "custom"
                },
                "width": 36
@@ -197,17 +197,17 @@
             {
                "format": "dollar",
                "help": {
-                  "link": "https://fac.gov/documentation/validation/#positive_number",
-                  "text": "The number must be zero or greater"
+                  "link": "https://fac.gov/documentation/validation/#any_number",
+                  "text": "Must be a number"
                },
                "range_name": "amount_expended",
                "title": "Amount Expended",
                "title_cell": "F1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "This cell must be a positive number",
-                  "custom_title": "Positive numbers",
-                  "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
+                  "custom_error": "This cell must be a number",
+                  "custom_title": "Numbers",
+                  "formula1": "=ISNUMBER(FIRSTCELLREF)",
                   "type": "custom"
                }
             },
@@ -259,37 +259,37 @@
                "format": "dollar",
                "formula": "=SUMIFS(amount_expended,cfda_key,V{0})",
                "help": {
-                  "link": "https://fac.gov/documentation/validation/#positive_number",
-                  "text": "The number must be zero or greater"
+                  "link": "https://fac.gov/documentation/validation/#any_number",
+                  "text": "Must be a number"
                },
                "keep_locked": true,
                "range_name": "federal_program_total",
                "title": "Federal Program Total",
                "title_cell": "J1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "This cell must be a positive number",
-                  "custom_title": "Positive numbers",
-                  "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
+                  "custom_error": "This cell must be a number",
+                  "custom_title": "Numbers",
+                  "formula1": "=ISNUMBER(FIRSTCELLREF)",
                   "type": "custom"
                }
             },
             {
                "format": "dollar",
                "formula": "=IF(G{0}=\"OTHER CLUSTER NOT LISTED ABOVE\",SUMIFS(amount_expended,uniform_other_cluster_name,X{0}), IF(AND(OR(G{0}=\"N/A\",G{0}=\"\"),H{0}=\"\"),0,IF(G{0}=\"STATE CLUSTER\",SUMIFS(amount_expended,uniform_state_cluster_name,W{0}),SUMIFS(amount_expended,cluster_name,G{0}))))",
                "help": {
-                  "link": "https://fac.gov/documentation/validation/#positive_number",
-                  "text": "The number must be zero or greater"
+                  "link": "https://fac.gov/documentation/validation/#any_number",
+                  "text": "Must be a number"
                },
                "keep_locked": true,
                "range_name": "cluster_total",
                "title": "Cluster Total",
                "title_cell": "K1",
                "type": "open_range",
                "validation": {
-                  "custom_error": "This cell must be a positive number",
-                  "custom_title": "Positive numbers",
-                  "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))",
+                  "custom_error": "This cell must be a number",
+                  "custom_title": "Numbers",
+                  "formula1": "=ISNUMBER(FIRSTCELLREF)",
                   "type": "custom"
                }
             },
@@ -393,8 +393,8 @@
             {
                "format": "dollar",
                "help": {
-                  "link": "https://fac.gov/documentation/validation/#positive_number",
-                  "text": "The number must be zero or greater"
+                  "link": "https://fac.gov/documentation/validation/#any_number",
+                  "text": "Must be a number"
                },
                "range_name": "subrecipient_amount",
                "title": "If yes (Passed Through), Amount Passed Through to Subrecipients",

diff --git a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx
diff --git a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx
diff --git a/backend/schemas/output/sections/FederalAwards.schema.json b/backend/schemas/output/sections/FederalAwards.schema.json
@@ -37,6 +37,12 @@
                      },
                      "cluster": {
                         "allOf": [
+                           {
+                              "required": [
+                                 "cluster_name",
+                                 "cluster_total"
+                              ]
+                           },
                            {
                               "if": {
                                  "properties": {
@@ -165,7 +171,6 @@
                               "type": "string"
                            },
                            "cluster_total": {
-                              "minimum": 0,
                               "type": "number"
                            },
                            "other_cluster_name": {
@@ -175,15 +180,16 @@
                               "type": "string"
                            }
                         },
-                        "required": [
-                           "cluster_name",
-                           "cluster_total"
-                        ],
                         "type": "object"
                      },
                      "direct_or_indirect_award": {
                         "additionalProperties": false,
                         "allOf": [
+                           {
+                              "required": [
+                                 "is_direct"
+                              ]
+                           },
                            {
                               "if": {
                                  "properties": {
@@ -243,14 +249,16 @@
                               "type": "string"
                            }
                         },
-                        "required": [
-                           "is_direct"
-                        ],
                         "type": "object"
                      },
                      "loan_or_loan_guarantee": {
                         "additionalProperties": false,
                         "allOf": [
+                           {
+                              "required": [
+                                 "is_guaranteed"
+                              ]
+                           },
                            {
                               "if": {
                                  "properties": {
@@ -307,14 +315,22 @@
                               ]
                            }
                         },
-                        "required": [
-                           "is_guaranteed"
-                        ],
                         "type": "object"
                      },
                      "program": {
                         "additionalProperties": false,
                         "allOf": [
+                           {
+                              "required": [
+                                 "program_name",
+                                 "federal_agency_prefix",
+                                 "three_digit_extension",
+                                 "is_major",
+                                 "number_of_audit_findings",
+                                 "federal_program_total",
+                                 "amount_expended"
+                              ]
+                           },
                            {
                               "properties": {
                                  "number_of_audit_findings": {
@@ -380,7 +396,6 @@
                               "type": "string"
                            },
                            "amount_expended": {
-                              "minimum": 0,
                               "type": "number"
                            },
                            "audit_report_type": {
@@ -407,7 +422,6 @@
                               "type": "string"
                            },
                            "federal_program_total": {
-                              "minimum": 0,
                               "type": "number"
                            },
                            "is_major": {
@@ -428,20 +442,16 @@
                               "type": "string"
                            }
                         },
-                        "required": [
-                           "program_name",
-                           "federal_agency_prefix",
-                           "three_digit_extension",
-                           "is_major",
-                           "number_of_audit_findings",
-                           "federal_program_total",
-                           "amount_expended"
-                        ],
                         "type": "object"
                      },
                      "subrecipients": {
                         "additionalProperties": false,
                         "allOf": [
+                           {
+                              "required": [
+                                 "is_passed"
+                              ]
+                           },
                            {
                               "if": {
                                  "properties": {
@@ -482,13 +492,9 @@
                               "type": "string"
                            },
                            "subrecipient_amount": {
-                              "minimum": 0,
                               "type": "number"
                            }
                         },
-                        "required": [
-                           "is_passed"
-                        ],
                         "type": "object"
                      }
                   },
@@ -505,7 +511,6 @@
                "type": "array"
             },
             "total_amount_expended": {
-               "minimum": 0,
                "type": "number"
             }
          },

diff --git a/backend/schemas/output/sections/TribalAccess.schema.json b/backend/schemas/output/sections/TribalAccess.schema.json
@@ -21,5 +21,5 @@
    ],
    "title": "TribalAccess",
    "type": "object",
-   "version": null
+   "version": 20230927
 }
diff --git a/backend/schemas/source/excel/libs/Help.libsonnet b/backend/schemas/source/excel/libs/Help.libsonnet
@@ -48,6 +48,10 @@ local make_url = function(anchor)
     text: 'The number must be zero or greater',
     link: make_url('positive_number'),
   },
+  any_number: {
+    text: 'Must be a number',
+    link: make_url('any_number'),
+  },
   prior_references: {
     text: 'Must be a comma-separated list of reference numbers (YYYY-NNN) or N/A.',
     link: make_url('prior_references'),

diff --git a/backend/schemas/source/excel/libs/SheetValidations.libsonnet b/backend/schemas/source/excel/libs/SheetValidations.libsonnet
@@ -52,6 +52,14 @@ local PositiveNumberValidation = {
   custom_title: 'Positive numbers',
 };
 
+local NumberValidation = {
+  type: 'custom',
+  // Is it a number ?
+  formula1: '=ISNUMBER(FIRSTCELLREF)',
+  custom_error: 'This cell must be a number',
+  custom_title: 'Numbers',
+};
+
 local ReferenceNumberValidation = {
   type: 'custom',
   //It is neccessary to allow blank otherwise user cannot delete the value
@@ -87,6 +95,7 @@ local AwardReferenceValidation = {
   NoValidation: { type: 'NOVALIDATION' },
   FAPPrefixValidation: FAPPrefixValidation,
   PositiveNumberValidation: PositiveNumberValidation,
+  NumberValidation: NumberValidation,
   LookupValidation: LookupValidation,
   RangeLookupValidation: RangeLookupValidation,
   StringOfLengthNine: StringOfSize(9),