diff --git a/backend/audit/cross_validation/errors.py b/backend/audit/cross_validation/errors.py index ba631fe44f..e31bffce2c 100644 --- a/backend/audit/cross_validation/errors.py +++ b/backend/audit/cross_validation/errors.py @@ -37,6 +37,10 @@ def err_missing_tribal_data_sharing_consent(): ) +def err_unexpected_tribal_data_sharing_consent(): + return "Tribal consent privacy flag set but non-tribal org type." + + def err_certifying_contacts_should_not_match(): return "The certifying auditor and auditee should not have the same email address." diff --git a/backend/audit/cross_validation/sac_validation_shape.py b/backend/audit/cross_validation/sac_validation_shape.py index eeb188ad56..76252a45bd 100644 --- a/backend/audit/cross_validation/sac_validation_shape.py +++ b/backend/audit/cross_validation/sac_validation_shape.py @@ -5,7 +5,7 @@ snake_to_camel, ) -at_root_sections = (NC.AUDIT_INFORMATION, NC.GENERAL_INFORMATION) # type: ignore +at_root_sections = (NC.AUDIT_INFORMATION, NC.GENERAL_INFORMATION, NC.TRIBAL_DATA_CONSENT) # type: ignore def get_shaped_section(sac, section_name): diff --git a/backend/audit/cross_validation/submission_progress_check.py b/backend/audit/cross_validation/submission_progress_check.py index d108c93a68..c61f9be124 100644 --- a/backend/audit/cross_validation/submission_progress_check.py +++ b/backend/audit/cross_validation/submission_progress_check.py @@ -31,14 +31,12 @@ def submission_progress_check(sac, sar=None, crossval=True): { "section_sname": [snake_case name of section], - "display": "hidden"/"incomplete"/"complete", + "display": "inactive"/"incomplete"/"complete", "completed": [bool], "completed_by": [email], "completed_date": [date], } """ - # TODO: remove these once tribal data consent are implemented: - del sac["sf_sac_sections"][NC.TRIBAL_DATA_CONSENT] # Add the status of the SAR into the list of sections: sac["sf_sac_sections"][NC.SINGLE_AUDIT_REPORT] = bool(sar) @@ -106,6 +104,9 @@ def get_num_findings(award): NC.ADDITIONAL_EINS: bool(general_info.get("multiple_eins_covered")), NC.SECONDARY_AUDITORS: bool(general_info.get("secondary_auditors_exist")), NC.SINGLE_AUDIT_REPORT: True, + NC.TRIBAL_DATA_CONSENT: bool( + general_info.get("user_provided_organization_type") == "tribal" + ), } # The General Information has its own condition, as it can be partially completed. diff --git a/backend/audit/cross_validation/test_tribal_data_sharing_consent.py b/backend/audit/cross_validation/test_tribal_data_sharing_consent.py index 19dfbd5c3d..844b1c6ecb 100644 --- a/backend/audit/cross_validation/test_tribal_data_sharing_consent.py +++ b/backend/audit/cross_validation/test_tribal_data_sharing_consent.py @@ -2,7 +2,10 @@ from audit.models import SingleAuditChecklist -from .errors import err_missing_tribal_data_sharing_consent +from .errors import ( + err_missing_tribal_data_sharing_consent, + err_unexpected_tribal_data_sharing_consent, +) from .tribal_data_sharing_consent import tribal_data_sharing_consent from .sac_validation_shape import sac_validation_shape @@ -53,11 +56,46 @@ def test_tribal_org_with_consent(self): sac.general_information = {"user_provided_organization_type": "tribal"} - shaped_sac = sac_validation_shape(sac) + sac.tribal_data_consent = { + "tribal_authorization_certifying_official_title": "Assistant Regional Manager", + "is_tribal_information_authorized_to_be_public": True, + "tribal_authorization_certifying_official_name": "A. Human", + } - # placeholder consent form until schema is finalized - shaped_sac["sf_sac_sections"]["tribal_data_consent"] = {"complete": True} + shaped_sac = sac_validation_shape(sac) validation_result = tribal_data_sharing_consent(shaped_sac) self.assertEqual(validation_result, []) + + def test_non_tribal_org_with_consent(self): + """SACS for non-tribal orgs should not pass if they have filled out a tribal consent form""" + sac = baker.make(SingleAuditChecklist) + + sac.tribal_data_consent = { + "tribal_authorization_certifying_official_title": "Assistant Regional Manager", + "is_tribal_information_authorized_to_be_public": True, + "tribal_authorization_certifying_official_name": "A. Human", + } + + non_tribal_org_types = [ + "state", + "local", + "higher-ed", + "non-profit", + "unknown", + "none", + ] + + for type in non_tribal_org_types: + with self.subTest(): + sac.general_information = {"user_provided_organization_type": type} + + shaped_sac = sac_validation_shape(sac) + + validation_result = tribal_data_sharing_consent(shaped_sac) + + self.assertEqual( + validation_result, + [{"error": err_unexpected_tribal_data_sharing_consent()}], + ) diff --git a/backend/audit/cross_validation/tribal_data_sharing_consent.py b/backend/audit/cross_validation/tribal_data_sharing_consent.py index 1133dd9413..7316610bd5 100644 --- a/backend/audit/cross_validation/tribal_data_sharing_consent.py +++ b/backend/audit/cross_validation/tribal_data_sharing_consent.py @@ -1,4 +1,7 @@ -from .errors import err_missing_tribal_data_sharing_consent +from .errors import ( + err_missing_tribal_data_sharing_consent, + err_unexpected_tribal_data_sharing_consent, +) def tribal_data_sharing_consent(sac_dict, *_args, **_kwargs): @@ -11,12 +14,33 @@ def tribal_data_sharing_consent(sac_dict, *_args, **_kwargs): "user_provided_organization_type" ) + required_fields = ( + "tribal_authorization_certifying_official_title", + "is_tribal_information_authorized_to_be_public", + "tribal_authorization_certifying_official_name", + ) + must_be_truthy_fields = ( + "tribal_authorization_certifying_official_title", + "tribal_authorization_certifying_official_name", + ) if organization_type == "tribal": if not (tribal_data_consent := all_sections.get("tribal_data_consent")): return [{"error": err_missing_tribal_data_sharing_consent()}] # this should check for consent form completeness once form data structure is finalized - if not tribal_data_consent["complete"]: + for rfield in required_fields: + if rfield not in tribal_data_consent: + return [{"error": err_missing_tribal_data_sharing_consent()}] + for tfield in must_be_truthy_fields: + if not tribal_data_consent.get(tfield): + return [{"error": err_missing_tribal_data_sharing_consent()}] + if not tribal_data_consent.get( + "is_tribal_information_authorized_to_be_public" + ) in (True, False): return [{"error": err_missing_tribal_data_sharing_consent()}] + # this shouldn't be possible now, but may be in the future + elif tc := all_sections.get("tribal_data_consent"): + if tc.get("is_tribal_information_authorized_to_be_public"): + return [{"error": err_unexpected_tribal_data_sharing_consent()}] return [] diff --git a/backend/audit/excel.py b/backend/audit/excel.py index fc99f3edc5..276e13f820 100644 --- a/backend/audit/excel.py +++ b/backend/audit/excel.py @@ -508,16 +508,54 @@ def _extract_column_data(workbook, result, params): set_fn(result, f"{parent_target}", entries) +def _has_only_one_field_with_value_0(my_dict, field_name): + """Check if the dictionary has exactly one field with the provided name and its value is 0""" + return len(my_dict) == 1 and my_dict.get(field_name) == 0 + + def _remove_empty_award_entries(data): """Removes empty award entries from the data""" - indexed_awards = [] + awards = [] for award in data.get("FederalAwards", {}).get("federal_awards", []): - if "program" in award: - program = award["program"] - if FEDERAL_AGENCY_PREFIX in program: - indexed_awards.append(award) + if not all( + [ + "direct_or_indirect_award" not in award, + "loan_or_loan_guarantee" not in award, + "subrecipients" not in award, + "program" in award + and _has_only_one_field_with_value_0( + award["program"], "federal_program_total" + ), + "cluster" in award + and _has_only_one_field_with_value_0(award["cluster"], "cluster_total"), + ] + ): + awards.append(award) if "FederalAwards" in data: # Update the federal_awards with the valid awards + data["FederalAwards"]["federal_awards"] = awards + + return data + + +def _add_required_fields(data): + """Adds empty parent fields to the json object to allow for proper schema validation / indexing""" + indexed_awards = [] + for award in data.get("FederalAwards", {}).get("federal_awards", []): + if "cluster" not in award: + award["cluster"] = {} + if "direct_or_indirect_award" not in award: + award["direct_or_indirect_award"] = {} + if "loan_or_loan_guarantee" not in award: + award["loan_or_loan_guarantee"] = {} + if "program" not in award: + award["program"] = {} + if "subrecipients" not in award: + award["subrecipients"] = {} + indexed_awards.append(award) + + if "FederalAwards" in data: + # Update the federal_awards with all required fields data["FederalAwards"]["federal_awards"] = indexed_awards return data @@ -536,7 +574,9 @@ def extract_federal_awards(file): template["title_row"], ) result = _extract_data(file, params) - return _remove_empty_award_entries(result) + result = _remove_empty_award_entries(result) + result = _add_required_fields(result) + return result def extract_corrective_action_plan(file): diff --git a/backend/audit/forms.py b/backend/audit/forms.py index cfbd766c9f..51c986c906 100644 --- a/backend/audit/forms.py +++ b/backend/audit/forms.py @@ -135,3 +135,23 @@ class AuditeeCertificationStep2Form(forms.Form): auditee_name = forms.CharField() auditee_title = forms.CharField() auditee_certification_date_signed = forms.DateField() + + +class TribalAuditConsentForm(forms.Form): + def clean_booleans(self): + data = self.cleaned_data + for k, v in data.items(): + if v == ["True"]: + data[k] = True + elif v == ["False"]: + data[k] = False + self.cleaned_data = data + return data + + choices_YoN = (("True", "Yes"), ("False", "No")) + + is_tribal_information_authorized_to_be_public = forms.MultipleChoiceField( + choices=choices_YoN + ) + tribal_authorization_certifying_official_name = forms.CharField() + tribal_authorization_certifying_official_title = forms.CharField() diff --git a/backend/audit/intake_to_dissemination.py b/backend/audit/intake_to_dissemination.py index 4be9e88a42..4edb4a2d5a 100644 --- a/backend/audit/intake_to_dissemination.py +++ b/backend/audit/intake_to_dissemination.py @@ -275,6 +275,7 @@ def load_general(self): submitted_date = self._convert_utc_to_american_samoa_zone( dates_by_status[status.SUBMITTED] ) + fac_accepted_date = submitted_date auditee_certify_name = auditee_certification.get("auditee_signature", {}).get( "auditee_name", "" ) @@ -365,10 +366,8 @@ def load_general(self): auditor_certified_date=auditor_certified_date, auditee_certified_date=auditee_certified_date, submitted_date=submitted_date, - # auditor_signature_date=auditor_certification["auditor_signature"]["auditor_certification_date_signed"], - # auditee_signature_date=auditee_certification["auditee_signature"]["auditee_certification_date_signed"], + fac_accepted_date=fac_accepted_date, audit_year=str(self.audit_year), - # is_duplicate_reports = Util.bool_to_yes_no(audit_information["is_aicpa_audit_guide_included"]), #FIXME This mapping does not seem correct total_amount_expended=total_amount_expended, type_audit_code="UG", is_public=is_public, diff --git a/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py b/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py new file mode 100644 index 0000000000..49f6c0c761 --- /dev/null +++ b/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.5 on 2023-10-03 19:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("audit", "0002_alter_singleauditchecklist_report_id"), + ] + + operations = [ + migrations.AlterField( + model_name="singleauditchecklist", + name="data_source", + field=models.CharField(default="GSAFAC"), + ), + migrations.AlterField( + model_name="submissionevent", + name="event", + field=models.CharField( + choices=[ + ("access-granted", "Access granted"), + ("additional-eins-updated", "Additional EINs updated"), + ("additional-ueis-updated", "Additional UEIs updated"), + ("audit-information-updated", "Audit information updated"), + ("audit-report-pdf-updated", "Audit report PDF updated"), + ( + "auditee-certification-completed", + "Auditee certification completed", + ), + ( + "auditor-certification-completed", + "Auditor certification completed", + ), + ( + "corrective-action-plan-updated", + "Corrective action plan updated", + ), + ("created", "Created"), + ("federal-awards-updated", "Federal awards updated"), + ( + "federal-awards-audit-findings-updated", + "Federal awards audit findings updated", + ), + ( + "federal-awards-audit-findings-text-updated", + "Federal awards audit findings text updated", + ), + ( + "findings-uniform-guidance-updated", + "Findings uniform guidance updated", + ), + ("general-information-updated", "General information updated"), + ("locked-for-certification", "Locked for certification"), + ("notes-to-sefa-updated", "Notes to SEFA updated"), + ("secondary-auditors-updated", "Secondary auditors updated"), + ("submitted", "Submitted to the FAC for processing"), + ("disseminated", "Copied to dissemination tables"), + ("tribal-consent-updated", "Tribal audit consent updated"), + ] + ), + ), + ] diff --git a/backend/audit/models.py b/backend/audit/models.py index 23c4c1e566..88019b1bac 100644 --- a/backend/audit/models.py +++ b/backend/audit/models.py @@ -563,10 +563,6 @@ def is_auditor_certified(self): def is_submitted(self): return self.submission_status in [SingleAuditChecklist.STATUS.SUBMITTED] - @property - def is_public(self): - return self.general_information["user_provided_organization_type"] != "tribal" - def get_transition_date(self, status): index = self.transition_name.index(status) if index >= 0: @@ -770,6 +766,8 @@ class EventType: NOTES_TO_SEFA_UPDATED = "notes-to-sefa-updated" SECONDARY_AUDITORS_UPDATED = "secondary-auditors-updated" SUBMITTED = "submitted" + DISSEMINATED = "disseminated" + TRIBAL_CONSENT_UPDATED = "tribal-consent-updated" EVENT_TYPES = ( (EventType.ACCESS_GRANTED, _("Access granted")), @@ -805,6 +803,8 @@ class EventType: (EventType.NOTES_TO_SEFA_UPDATED, _("Notes to SEFA updated")), (EventType.SECONDARY_AUDITORS_UPDATED, _("Secondary auditors updated")), (EventType.SUBMITTED, _("Submitted to the FAC for processing")), + (EventType.DISSEMINATED, _("Copied to dissemination tables")), + (EventType.TRIBAL_CONSENT_UPDATED, _("Tribal audit consent updated")), ) sac = models.ForeignKey(SingleAuditChecklist, on_delete=models.CASCADE) diff --git a/backend/audit/templates/audit/no-late-changes.html b/backend/audit/templates/audit/no-late-changes.html index 1c6b61175b..225736db2c 100644 --- a/backend/audit/templates/audit/no-late-changes.html +++ b/backend/audit/templates/audit/no-late-changes.html @@ -6,7 +6,7 @@