diff --git a/backend/audit/cross_validation/errors.py b/backend/audit/cross_validation/errors.py index ba631fe44f..e31bffce2c 100644 --- a/backend/audit/cross_validation/errors.py +++ b/backend/audit/cross_validation/errors.py @@ -37,6 +37,10 @@ def err_missing_tribal_data_sharing_consent(): ) +def err_unexpected_tribal_data_sharing_consent(): + return "Tribal consent privacy flag set but non-tribal org type." + + def err_certifying_contacts_should_not_match(): return "The certifying auditor and auditee should not have the same email address." diff --git a/backend/audit/cross_validation/sac_validation_shape.py b/backend/audit/cross_validation/sac_validation_shape.py index eeb188ad56..76252a45bd 100644 --- a/backend/audit/cross_validation/sac_validation_shape.py +++ b/backend/audit/cross_validation/sac_validation_shape.py @@ -5,7 +5,7 @@ snake_to_camel, ) -at_root_sections = (NC.AUDIT_INFORMATION, NC.GENERAL_INFORMATION) # type: ignore +at_root_sections = (NC.AUDIT_INFORMATION, NC.GENERAL_INFORMATION, NC.TRIBAL_DATA_CONSENT) # type: ignore def get_shaped_section(sac, section_name): diff --git a/backend/audit/cross_validation/submission_progress_check.py b/backend/audit/cross_validation/submission_progress_check.py index d108c93a68..c61f9be124 100644 --- a/backend/audit/cross_validation/submission_progress_check.py +++ b/backend/audit/cross_validation/submission_progress_check.py @@ -31,14 +31,12 @@ def submission_progress_check(sac, sar=None, crossval=True): { "section_sname": [snake_case name of section], - "display": "hidden"/"incomplete"/"complete", + "display": "inactive"/"incomplete"/"complete", "completed": [bool], "completed_by": [email], "completed_date": [date], } """ - # TODO: remove these once tribal data consent are implemented: - del sac["sf_sac_sections"][NC.TRIBAL_DATA_CONSENT] # Add the status of the SAR into the list of sections: sac["sf_sac_sections"][NC.SINGLE_AUDIT_REPORT] = bool(sar) @@ -106,6 +104,9 @@ def get_num_findings(award): NC.ADDITIONAL_EINS: bool(general_info.get("multiple_eins_covered")), NC.SECONDARY_AUDITORS: bool(general_info.get("secondary_auditors_exist")), NC.SINGLE_AUDIT_REPORT: True, + NC.TRIBAL_DATA_CONSENT: bool( + general_info.get("user_provided_organization_type") == "tribal" + ), } # The General Information has its own condition, as it can be partially completed. diff --git a/backend/audit/cross_validation/test_tribal_data_sharing_consent.py b/backend/audit/cross_validation/test_tribal_data_sharing_consent.py index 19dfbd5c3d..844b1c6ecb 100644 --- a/backend/audit/cross_validation/test_tribal_data_sharing_consent.py +++ b/backend/audit/cross_validation/test_tribal_data_sharing_consent.py @@ -2,7 +2,10 @@ from audit.models import SingleAuditChecklist -from .errors import err_missing_tribal_data_sharing_consent +from .errors import ( + err_missing_tribal_data_sharing_consent, + err_unexpected_tribal_data_sharing_consent, +) from .tribal_data_sharing_consent import tribal_data_sharing_consent from .sac_validation_shape import sac_validation_shape @@ -53,11 +56,46 @@ def test_tribal_org_with_consent(self): sac.general_information = {"user_provided_organization_type": "tribal"} - shaped_sac = sac_validation_shape(sac) + sac.tribal_data_consent = { + "tribal_authorization_certifying_official_title": "Assistant Regional Manager", + "is_tribal_information_authorized_to_be_public": True, + "tribal_authorization_certifying_official_name": "A. Human", + } - # placeholder consent form until schema is finalized - shaped_sac["sf_sac_sections"]["tribal_data_consent"] = {"complete": True} + shaped_sac = sac_validation_shape(sac) validation_result = tribal_data_sharing_consent(shaped_sac) self.assertEqual(validation_result, []) + + def test_non_tribal_org_with_consent(self): + """SACS for non-tribal orgs should not pass if they have filled out a tribal consent form""" + sac = baker.make(SingleAuditChecklist) + + sac.tribal_data_consent = { + "tribal_authorization_certifying_official_title": "Assistant Regional Manager", + "is_tribal_information_authorized_to_be_public": True, + "tribal_authorization_certifying_official_name": "A. Human", + } + + non_tribal_org_types = [ + "state", + "local", + "higher-ed", + "non-profit", + "unknown", + "none", + ] + + for type in non_tribal_org_types: + with self.subTest(): + sac.general_information = {"user_provided_organization_type": type} + + shaped_sac = sac_validation_shape(sac) + + validation_result = tribal_data_sharing_consent(shaped_sac) + + self.assertEqual( + validation_result, + [{"error": err_unexpected_tribal_data_sharing_consent()}], + ) diff --git a/backend/audit/cross_validation/tribal_data_sharing_consent.py b/backend/audit/cross_validation/tribal_data_sharing_consent.py index 1133dd9413..7316610bd5 100644 --- a/backend/audit/cross_validation/tribal_data_sharing_consent.py +++ b/backend/audit/cross_validation/tribal_data_sharing_consent.py @@ -1,4 +1,7 @@ -from .errors import err_missing_tribal_data_sharing_consent +from .errors import ( + err_missing_tribal_data_sharing_consent, + err_unexpected_tribal_data_sharing_consent, +) def tribal_data_sharing_consent(sac_dict, *_args, **_kwargs): @@ -11,12 +14,33 @@ def tribal_data_sharing_consent(sac_dict, *_args, **_kwargs): "user_provided_organization_type" ) + required_fields = ( + "tribal_authorization_certifying_official_title", + "is_tribal_information_authorized_to_be_public", + "tribal_authorization_certifying_official_name", + ) + must_be_truthy_fields = ( + "tribal_authorization_certifying_official_title", + "tribal_authorization_certifying_official_name", + ) if organization_type == "tribal": if not (tribal_data_consent := all_sections.get("tribal_data_consent")): return [{"error": err_missing_tribal_data_sharing_consent()}] # this should check for consent form completeness once form data structure is finalized - if not tribal_data_consent["complete"]: + for rfield in required_fields: + if rfield not in tribal_data_consent: + return [{"error": err_missing_tribal_data_sharing_consent()}] + for tfield in must_be_truthy_fields: + if not tribal_data_consent.get(tfield): + return [{"error": err_missing_tribal_data_sharing_consent()}] + if not tribal_data_consent.get( + "is_tribal_information_authorized_to_be_public" + ) in (True, False): return [{"error": err_missing_tribal_data_sharing_consent()}] + # this shouldn't be possible now, but may be in the future + elif tc := all_sections.get("tribal_data_consent"): + if tc.get("is_tribal_information_authorized_to_be_public"): + return [{"error": err_unexpected_tribal_data_sharing_consent()}] return [] diff --git a/backend/audit/excel.py b/backend/audit/excel.py index fc99f3edc5..276e13f820 100644 --- a/backend/audit/excel.py +++ b/backend/audit/excel.py @@ -508,16 +508,54 @@ def _extract_column_data(workbook, result, params): set_fn(result, f"{parent_target}", entries) +def _has_only_one_field_with_value_0(my_dict, field_name): + """Check if the dictionary has exactly one field with the provided name and its value is 0""" + return len(my_dict) == 1 and my_dict.get(field_name) == 0 + + def _remove_empty_award_entries(data): """Removes empty award entries from the data""" - indexed_awards = [] + awards = [] for award in data.get("FederalAwards", {}).get("federal_awards", []): - if "program" in award: - program = award["program"] - if FEDERAL_AGENCY_PREFIX in program: - indexed_awards.append(award) + if not all( + [ + "direct_or_indirect_award" not in award, + "loan_or_loan_guarantee" not in award, + "subrecipients" not in award, + "program" in award + and _has_only_one_field_with_value_0( + award["program"], "federal_program_total" + ), + "cluster" in award + and _has_only_one_field_with_value_0(award["cluster"], "cluster_total"), + ] + ): + awards.append(award) if "FederalAwards" in data: # Update the federal_awards with the valid awards + data["FederalAwards"]["federal_awards"] = awards + + return data + + +def _add_required_fields(data): + """Adds empty parent fields to the json object to allow for proper schema validation / indexing""" + indexed_awards = [] + for award in data.get("FederalAwards", {}).get("federal_awards", []): + if "cluster" not in award: + award["cluster"] = {} + if "direct_or_indirect_award" not in award: + award["direct_or_indirect_award"] = {} + if "loan_or_loan_guarantee" not in award: + award["loan_or_loan_guarantee"] = {} + if "program" not in award: + award["program"] = {} + if "subrecipients" not in award: + award["subrecipients"] = {} + indexed_awards.append(award) + + if "FederalAwards" in data: + # Update the federal_awards with all required fields data["FederalAwards"]["federal_awards"] = indexed_awards return data @@ -536,7 +574,9 @@ def extract_federal_awards(file): template["title_row"], ) result = _extract_data(file, params) - return _remove_empty_award_entries(result) + result = _remove_empty_award_entries(result) + result = _add_required_fields(result) + return result def extract_corrective_action_plan(file): diff --git a/backend/audit/forms.py b/backend/audit/forms.py index cfbd766c9f..51c986c906 100644 --- a/backend/audit/forms.py +++ b/backend/audit/forms.py @@ -135,3 +135,23 @@ class AuditeeCertificationStep2Form(forms.Form): auditee_name = forms.CharField() auditee_title = forms.CharField() auditee_certification_date_signed = forms.DateField() + + +class TribalAuditConsentForm(forms.Form): + def clean_booleans(self): + data = self.cleaned_data + for k, v in data.items(): + if v == ["True"]: + data[k] = True + elif v == ["False"]: + data[k] = False + self.cleaned_data = data + return data + + choices_YoN = (("True", "Yes"), ("False", "No")) + + is_tribal_information_authorized_to_be_public = forms.MultipleChoiceField( + choices=choices_YoN + ) + tribal_authorization_certifying_official_name = forms.CharField() + tribal_authorization_certifying_official_title = forms.CharField() diff --git a/backend/audit/intake_to_dissemination.py b/backend/audit/intake_to_dissemination.py index 4be9e88a42..4edb4a2d5a 100644 --- a/backend/audit/intake_to_dissemination.py +++ b/backend/audit/intake_to_dissemination.py @@ -275,6 +275,7 @@ def load_general(self): submitted_date = self._convert_utc_to_american_samoa_zone( dates_by_status[status.SUBMITTED] ) + fac_accepted_date = submitted_date auditee_certify_name = auditee_certification.get("auditee_signature", {}).get( "auditee_name", "" ) @@ -365,10 +366,8 @@ def load_general(self): auditor_certified_date=auditor_certified_date, auditee_certified_date=auditee_certified_date, submitted_date=submitted_date, - # auditor_signature_date=auditor_certification["auditor_signature"]["auditor_certification_date_signed"], - # auditee_signature_date=auditee_certification["auditee_signature"]["auditee_certification_date_signed"], + fac_accepted_date=fac_accepted_date, audit_year=str(self.audit_year), - # is_duplicate_reports = Util.bool_to_yes_no(audit_information["is_aicpa_audit_guide_included"]), #FIXME This mapping does not seem correct total_amount_expended=total_amount_expended, type_audit_code="UG", is_public=is_public, diff --git a/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py b/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py new file mode 100644 index 0000000000..49f6c0c761 --- /dev/null +++ b/backend/audit/migrations/0003_alter_singleauditchecklist_data_source_and_more.py @@ -0,0 +1,63 @@ +# Generated by Django 4.2.5 on 2023-10-03 19:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("audit", "0002_alter_singleauditchecklist_report_id"), + ] + + operations = [ + migrations.AlterField( + model_name="singleauditchecklist", + name="data_source", + field=models.CharField(default="GSAFAC"), + ), + migrations.AlterField( + model_name="submissionevent", + name="event", + field=models.CharField( + choices=[ + ("access-granted", "Access granted"), + ("additional-eins-updated", "Additional EINs updated"), + ("additional-ueis-updated", "Additional UEIs updated"), + ("audit-information-updated", "Audit information updated"), + ("audit-report-pdf-updated", "Audit report PDF updated"), + ( + "auditee-certification-completed", + "Auditee certification completed", + ), + ( + "auditor-certification-completed", + "Auditor certification completed", + ), + ( + "corrective-action-plan-updated", + "Corrective action plan updated", + ), + ("created", "Created"), + ("federal-awards-updated", "Federal awards updated"), + ( + "federal-awards-audit-findings-updated", + "Federal awards audit findings updated", + ), + ( + "federal-awards-audit-findings-text-updated", + "Federal awards audit findings text updated", + ), + ( + "findings-uniform-guidance-updated", + "Findings uniform guidance updated", + ), + ("general-information-updated", "General information updated"), + ("locked-for-certification", "Locked for certification"), + ("notes-to-sefa-updated", "Notes to SEFA updated"), + ("secondary-auditors-updated", "Secondary auditors updated"), + ("submitted", "Submitted to the FAC for processing"), + ("disseminated", "Copied to dissemination tables"), + ("tribal-consent-updated", "Tribal audit consent updated"), + ] + ), + ), + ] diff --git a/backend/audit/models.py b/backend/audit/models.py index 23c4c1e566..88019b1bac 100644 --- a/backend/audit/models.py +++ b/backend/audit/models.py @@ -563,10 +563,6 @@ def is_auditor_certified(self): def is_submitted(self): return self.submission_status in [SingleAuditChecklist.STATUS.SUBMITTED] - @property - def is_public(self): - return self.general_information["user_provided_organization_type"] != "tribal" - def get_transition_date(self, status): index = self.transition_name.index(status) if index >= 0: @@ -770,6 +766,8 @@ class EventType: NOTES_TO_SEFA_UPDATED = "notes-to-sefa-updated" SECONDARY_AUDITORS_UPDATED = "secondary-auditors-updated" SUBMITTED = "submitted" + DISSEMINATED = "disseminated" + TRIBAL_CONSENT_UPDATED = "tribal-consent-updated" EVENT_TYPES = ( (EventType.ACCESS_GRANTED, _("Access granted")), @@ -805,6 +803,8 @@ class EventType: (EventType.NOTES_TO_SEFA_UPDATED, _("Notes to SEFA updated")), (EventType.SECONDARY_AUDITORS_UPDATED, _("Secondary auditors updated")), (EventType.SUBMITTED, _("Submitted to the FAC for processing")), + (EventType.DISSEMINATED, _("Copied to dissemination tables")), + (EventType.TRIBAL_CONSENT_UPDATED, _("Tribal audit consent updated")), ) sac = models.ForeignKey(SingleAuditChecklist, on_delete=models.CASCADE) diff --git a/backend/audit/templates/audit/no-late-changes.html b/backend/audit/templates/audit/no-late-changes.html index 1c6b61175b..225736db2c 100644 --- a/backend/audit/templates/audit/no-late-changes.html +++ b/backend/audit/templates/audit/no-late-changes.html @@ -6,7 +6,7 @@
{% csrf_token %}
- + Access denied

diff --git a/backend/audit/templates/audit/submission_checklist/submission-checklist.html b/backend/audit/templates/audit/submission_checklist/submission-checklist.html index 1eeea2ef12..cbc7ca40f4 100644 --- a/backend/audit/templates/audit/submission_checklist/submission-checklist.html +++ b/backend/audit/templates/audit/submission_checklist/submission-checklist.html @@ -39,6 +39,67 @@ + + {% comment %} Tribal data release consent {% endcomment %} + {% comment %} + Five display cases: + 1. Does not appear at all for non-tribal entities. + 2. The auditee certifying official sees the link and can complete it. + 3. The form is complete. The auditee certifying official can edit it. + 4. Anyone else with access sees an inactive section, no link. + 5. The form is complete. Anyone else with access sees that it's done but still can't click a link. + {% endcomment %} + {% comment %} 1. Does not appear at all for non-tribal entities. {% endcomment %} + {% if user_provided_organization_type == "tribal" %} +

+ {% endif %} {% comment %} Pre-submission validation {% endcomment %}
  • diff --git a/backend/audit/templates/audit/tribal-data-release.html b/backend/audit/templates/audit/tribal-data-release.html new file mode 100644 index 0000000000..3fa3437dbf --- /dev/null +++ b/backend/audit/templates/audit/tribal-data-release.html @@ -0,0 +1,111 @@ +{% extends "base.html" %} +{% load static %} +{% block content %} +
    +
    + + {% csrf_token %} +
    +

    Tribal data release

    +

    + Determine if the audit should make all information contained in the form SF-SAC and reporting package publicly available. +

    + +
    +

    + If you suppress your reporting package, you must submit your reporting package directly to: +

    +
      +
    • + any pass-through entities that awarded Federal funds; and +
    • +
    • + any pass-through entities with findings related to Federal awards listed on the summary schedule of prior audit findings +
    • +
    +

    + For the full citation, see Uniform Guidance 2 CFR 200.512(b)(2) +

    +
    + + {% comment %} This div is the grey box, and should handle its own margins and padding. {% endcomment %} +
    +
    + + I certify that, to the best of my knowledge and belief, the entity has: + +
    + + +
    +
    + + +
    + {{ errors.is_tribal_information_authorized_to_be_public|striptags }} +
    + +

    Agree and sign Tribal data release

    +
    +
    + + + {{ errors.tribal_authorization_certifying_official_name|striptags }} +
    + +
    + + + {{ errors.tribal_authorization_certifying_official_title|striptags }} +
    + +
    + + Cancel +
    +
    +
    +
    + +
    +
    + {% include "audit-metadata.html" %} +{% endblock content %} diff --git a/backend/audit/test_views.py b/backend/audit/test_views.py index a6dec15473..14af327825 100644 --- a/backend/audit/test_views.py +++ b/backend/audit/test_views.py @@ -185,7 +185,7 @@ class SubmissionViewTests(TestCase): def test_post_redirect(self): """ - The status should be "submitted" after the post. + The status should be "disseminated" after the post. The user should be redirected to the submissions table. """ filename = "general-information--test0001test--simple-pass.json" @@ -233,7 +233,7 @@ def test_post_redirect(self): ) sac_after = SingleAuditChecklist.objects.get(report_id=sac.report_id) self.assertEqual(response.status_code, 302) - self.assertEqual(sac_after.submission_status, sac_after.STATUS.SUBMITTED) + self.assertEqual(sac_after.submission_status, sac_after.STATUS.DISSEMINATED) class SubmissionStatusTests(TestCase): @@ -455,7 +455,7 @@ def test_submission(self): updated_sac = SingleAuditChecklist.objects.get(report_id=sac.report_id) - self.assertEqual(updated_sac.submission_status, "submitted") + self.assertEqual(updated_sac.submission_status, "disseminated") submission_events = SubmissionEvent.objects.filter(sac=sac) @@ -464,7 +464,7 @@ def test_submission(self): self.assertGreaterEqual(event_count, 1) self.assertEqual( submission_events[event_count - 1].event, - SubmissionEvent.EventType.SUBMITTED, + SubmissionEvent.EventType.DISSEMINATED, ) diff --git a/backend/audit/urls.py b/backend/audit/urls.py index fc821cf45a..9adeeccdc7 100644 --- a/backend/audit/urls.py +++ b/backend/audit/urls.py @@ -70,6 +70,11 @@ def camel_to_hyphen(raw: str) -> str: views.UploadReportView.as_view(), name="UploadReport", ), + path( + "tribal-data-release/", + views.TribalDataConsent.as_view(), + name="TribalAuditConsent", + ), path( "cross-validation/", views.CrossValidationView.as_view(), diff --git a/backend/audit/validators.py b/backend/audit/validators.py index 769d522bed..30baca6df8 100644 --- a/backend/audit/validators.py +++ b/backend/audit/validators.py @@ -3,7 +3,6 @@ import logging from jsonschema import Draft7Validator, FormatChecker, validate from jsonschema.exceptions import ValidationError as JSONSchemaValidationError -from jsonschema.exceptions import SchemaError as JSONSchemaError from django.core.exceptions import ValidationError @@ -209,18 +208,8 @@ def validate_federal_award_json(value): schema_path = settings.SECTION_SCHEMA_DIR / "FederalAwards.schema.json" schema = json.loads(schema_path.read_text(encoding="utf-8")) - # FIXME: For some classes of error, this approach - # will go into an infinite/recursive loop in the iterator. - # There needs to either be a way to solve that, or we need - # to not use this approach. - # validator = Draft7Validator(schema) - # errors = list(validator.iter_errors(value)) - # The side-effect is that now I only do one error at a time... - errors = [] - try: - validate(schema=schema, instance=value) - except (JSONSchemaValidationError, JSONSchemaError) as e: - errors = [e] + validator = Draft7Validator(schema) + errors = list(validator.iter_errors(value)) if len(errors) > 0: raise ValidationError(message=_federal_awards_json_error(errors)) diff --git a/backend/audit/viewlib/__init__.py b/backend/audit/viewlib/__init__.py index 99d9073af0..e1d3330410 100644 --- a/backend/audit/viewlib/__init__.py +++ b/backend/audit/viewlib/__init__.py @@ -2,6 +2,7 @@ SubmissionProgressView, submission_progress_check, ) +from .tribal_data_consent import TribalDataConsent from .upload_report_view import UploadReportView @@ -9,4 +10,5 @@ views = [ SubmissionProgressView, UploadReportView, + TribalDataConsent, ] diff --git a/backend/audit/viewlib/submission_progress_view.py b/backend/audit/viewlib/submission_progress_view.py index 44abcf9474..4eed9f8485 100644 --- a/backend/audit/viewlib/submission_progress_view.py +++ b/backend/audit/viewlib/submission_progress_view.py @@ -9,10 +9,7 @@ from audit.mixins import ( SingleAuditChecklistAccessRequiredMixin, ) -from audit.models import ( - SingleAuditChecklist, - SingleAuditReportFile, -) +from audit.models import SingleAuditChecklist, SingleAuditReportFile, Access # Turn the named tuples into dicts because Django templates work with dicts: @@ -90,7 +87,7 @@ class SubmissionProgressView(SingleAuditChecklistAccessRequiredMixin, generic.Vi The states are: - + hidden + + inactive + incomplete + complete @@ -103,6 +100,20 @@ def get(self, request, *args, **kwargs): try: sac = SingleAuditChecklist.objects.get(report_id=report_id) + + # Determine if the auditee certifier is the same as the current user. + # If there is no auditee certifier, default to False. + is_user_auditee_certifier = False + sac_auditee_results = Access.objects.filter( + sac_id=sac.id, role="certifying_auditee_contact" + ).values() # ValuesQuerySet (array of dicts) + if sac_auditee_results.exists(): + is_user_auditee_certifier = ( + sac_auditee_results[0].get("user_id") == request.user.id + ) + + is_tribal_data_consent_complete = True if sac.tribal_data_consent else False + try: sar = SingleAuditReportFile.objects.filter(sac_id=sac.id).latest( "date_created" @@ -149,6 +160,8 @@ def get(self, request, *args, **kwargs): "auditee_name": sac.auditee_name, "auditee_uei": sac.auditee_uei, "user_provided_organization_type": sac.user_provided_organization_type, + "is_user_auditee_certifier": is_user_auditee_certifier, + "is_tribal_data_consent_complete": is_tribal_data_consent_complete, } context = context | subcheck diff --git a/backend/audit/viewlib/tribal_data_consent.py b/backend/audit/viewlib/tribal_data_consent.py new file mode 100644 index 0000000000..ea6f130c44 --- /dev/null +++ b/backend/audit/viewlib/tribal_data_consent.py @@ -0,0 +1,79 @@ +import logging + +from django.core.exceptions import PermissionDenied +from django.shortcuts import render, redirect +from django.views import generic +from django.urls import reverse +from audit.mixins import ( + SingleAuditChecklistAccessRequiredMixin, +) +from audit.models import ( + SingleAuditChecklist, + SubmissionEvent, +) +from audit.forms import TribalAuditConsentForm +from audit.validators import validate_tribal_data_consent_json + + +logger = logging.getLogger(__name__) + + +class TribalDataConsent(SingleAuditChecklistAccessRequiredMixin, generic.View): + def get(self, request, *args, **kwargs): + report_id = kwargs["report_id"] + + try: + sac = SingleAuditChecklist.objects.get(report_id=report_id) + tribal_audit_consent = sac.tribal_data_consent or {} + + context = { + "auditee_uei": sac.auditee_uei, + "auditee_name": sac.auditee_name, + "report_id": report_id, + } + + return render( + request, + "audit/tribal-data-release.html", + context | tribal_audit_consent, + ) + except SingleAuditChecklist.DoesNotExist: + raise PermissionDenied("You do not have access to this audit.") + + def post(self, request, *args, **kwargs): + report_id = kwargs["report_id"] + + try: + sac = SingleAuditChecklist.objects.get(report_id=report_id) + form = TribalAuditConsentForm(request.POST or None) + + if form.is_valid(): + form.clean_booleans() + tribal_data_consent = form.cleaned_data + validated = validate_tribal_data_consent_json(tribal_data_consent) + sac.tribal_data_consent = validated + sac.save( + event_user=request.user, + event_type=SubmissionEvent.EventType.TRIBAL_CONSENT_UPDATED, + ) + logger.info("Tribal data consent saved.", tribal_data_consent) + + return redirect(reverse("audit:SubmissionProgress", args=[report_id])) + + tribal_data_consent = sac.tribal_data_consent or {} + + context = { + "auditee_uei": sac.auditee_uei, + "auditee_name": sac.auditee_name, + "report_id": report_id, + "errors": form.errors, + } + + return render( + request, + "audit/tribal-data-release.html", + context | tribal_data_consent, + ) + + except SingleAuditChecklist.DoesNotExist: + raise PermissionDenied("You do not have access to this audit.") diff --git a/backend/audit/views.py b/backend/audit/views.py index 0d35d1af90..7668883b66 100644 --- a/backend/audit/views.py +++ b/backend/audit/views.py @@ -64,6 +64,7 @@ validate_secondary_auditors_json, ) from audit.viewlib import ( # noqa + TribalDataConsent, SubmissionProgressView, UploadReportView, submission_progress_check, @@ -94,7 +95,7 @@ def get(self, request, *args, **kwargs): audit["submission_status"] = ( audit["submission_status"].replace("_", " ").title() ) # auditee_certified --> Auditee Certified - if audit["submission_status"] == "Submitted": + if audit["submission_status"] in ["Submitted", "Disseminated"]: data["completed_audits"].append(audit) else: data["in_progress_audits"].append(audit) @@ -722,6 +723,10 @@ def post(self, request, *args, **kwargs): # FIXME: We should now provide a reasonable error to the user. if disseminated is None: sac.transition_to_disseminated() + sac.save( + event_user=request.user, + event_type=SubmissionEvent.EventType.DISSEMINATED, + ) logger.info( "Dissemination errors: %s, report_id: %s", disseminated, report_id diff --git a/backend/cypress/e2e/full-submission.cy.js b/backend/cypress/e2e/full-submission.cy.js index 1c02d57c1f..08c215b98b 100644 --- a/backend/cypress/e2e/full-submission.cy.js +++ b/backend/cypress/e2e/full-submission.cy.js @@ -10,6 +10,8 @@ import { testPdfAuditReport } from '../support/report-pdf.js'; import { testAuditorCertification } from '../support/auditor-certification.js'; import { testAuditeeCertification } from '../support/auditee-certification.js'; import { testReportIdFound, testReportIdNotFound } from '../support/dissemination-table.js'; +import { testTribalAuditPublic, testTribalAuditPrivate } from '../support/tribal-audit-form.js'; + import { testWorkbookFederalAwards, testWorkbookNotesToSEFA, @@ -24,7 +26,6 @@ import { const LOGIN_TEST_EMAIL_AUDITEE = Cypress.env('LOGIN_TEST_EMAIL_AUDITEE'); const LOGIN_TEST_PASSWORD_AUDITEE = Cypress.env('LOGIN_TEST_PASSWORD_AUDITEE'); const LOGIN_TEST_OTP_SECRET_AUDITEE = Cypress.env('LOGIN_TEST_OTP_SECRET_AUDITEE'); -const API_GOV_JWT = Cypress.env('API_GOV_JWT'); describe('Full audit submission', () => { before(() => { @@ -92,6 +93,28 @@ describe('Full audit submission', () => { cy.get(".usa-link").contains("Additional EINs").click(); testWorkbookAdditionalEINs(false); + cy.url().then(url => { + const reportId = url.split('/').pop(); + + // Login as Auditee + testLogoutGov(); + testLoginGovLogin( + LOGIN_TEST_EMAIL_AUDITEE, + LOGIN_TEST_PASSWORD_AUDITEE, + LOGIN_TEST_OTP_SECRET_AUDITEE + ); + cy.visit(`/audit/submission-progress/${reportId}`); + + // complete the tribal audit form as auditee - opt private + cy.get(".usa-link").contains("Tribal data release").click(); + testTribalAuditPrivate(); + + // Login as Auditor + testLogoutGov(); + testLoginGovLogin(); + cy.visit(`/audit/submission-progress/${reportId}`); + }) + // Complete the audit information form cy.get(".usa-link").contains("Audit Information form").click(); testAuditInformationForm(); @@ -124,7 +147,7 @@ describe('Full audit submission', () => { // Submit cy.get(".usa-link").contains("Submit to the FAC for processing").click(); - cy.url().should('match', /\/audit\/submission\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/submission\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); cy.get('#continue').click(); cy.url().should('match', /\/audit\//); @@ -134,8 +157,8 @@ describe('Full audit submission', () => { 'The audits listed below have been submitted to the FAC for processing and may not be edited.', ).siblings().contains('td', reportId); - // Report should now be in the dissemination table - testReportIdFound(reportId); + // The Report should not be in the dissemination table + testReportIdNotFound(reportId); }); }); }); diff --git a/backend/cypress/fixtures/additional-ueis-template.xlsx b/backend/cypress/fixtures/additional-ueis-template.xlsx deleted file mode 100644 index 6c719b713a..0000000000 Binary files a/backend/cypress/fixtures/additional-ueis-template.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/additional-ueis.xlsx b/backend/cypress/fixtures/additional-ueis.xlsx deleted file mode 100644 index 94930f5227..0000000000 Binary files a/backend/cypress/fixtures/additional-ueis.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/audit-findings-text-UPDATE.xlsx b/backend/cypress/fixtures/audit-findings-text-UPDATE.xlsx deleted file mode 100644 index f37ce9d6a2..0000000000 Binary files a/backend/cypress/fixtures/audit-findings-text-UPDATE.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/cap-invalid.xlsx b/backend/cypress/fixtures/cap-invalid.xlsx deleted file mode 100644 index 58856bcff1..0000000000 Binary files a/backend/cypress/fixtures/cap-invalid.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/corrective-action-plan-UPDATE.xlsx b/backend/cypress/fixtures/corrective-action-plan-UPDATE.xlsx deleted file mode 100644 index 6d969b2280..0000000000 Binary files a/backend/cypress/fixtures/corrective-action-plan-UPDATE.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/fed-awards-invalid.xlsx b/backend/cypress/fixtures/fed-awards-invalid.xlsx deleted file mode 100644 index 374022a241..0000000000 Binary files a/backend/cypress/fixtures/fed-awards-invalid.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/federal-awards-UPDATE.xlsx b/backend/cypress/fixtures/federal-awards-UPDATE.xlsx deleted file mode 100644 index 796b6a063e..0000000000 Binary files a/backend/cypress/fixtures/federal-awards-UPDATE.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/federal-awards-audit-findings-UPDATE.xlsx b/backend/cypress/fixtures/federal-awards-audit-findings-UPDATE.xlsx deleted file mode 100644 index 28e6c6ca5c..0000000000 Binary files a/backend/cypress/fixtures/federal-awards-audit-findings-UPDATE.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/find-text-invalid.xlsx b/backend/cypress/fixtures/find-text-invalid.xlsx deleted file mode 100644 index cbe1b8d913..0000000000 Binary files a/backend/cypress/fixtures/find-text-invalid.xlsx and /dev/null differ diff --git a/backend/cypress/fixtures/find-uni-invalid.xlsx b/backend/cypress/fixtures/find-uni-invalid.xlsx deleted file mode 100644 index d5ec4eee8c..0000000000 Binary files a/backend/cypress/fixtures/find-uni-invalid.xlsx and /dev/null differ diff --git a/backend/cypress/support/audit-info-form.js b/backend/cypress/support/audit-info-form.js index c4cca71fb2..623b9c85e6 100644 --- a/backend/cypress/support/audit-info-form.js +++ b/backend/cypress/support/audit-info-form.js @@ -33,5 +33,5 @@ export function testAuditInformationForm() { cy.get('.usa-button').contains('Save and continue').click({force: true}); - cy.url().should('match', /\/audit\/submission-progress\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); } \ No newline at end of file diff --git a/backend/cypress/support/check-eligibility.js b/backend/cypress/support/check-eligibility.js index f930b10bbf..e941c20b76 100644 --- a/backend/cypress/support/check-eligibility.js +++ b/backend/cypress/support/check-eligibility.js @@ -1,7 +1,7 @@ // Resusable components for the "Check Eligibility" pre-screener export function selectValidEntries() { - cy.get('label[for=entity-state]').click(); + cy.get('label[for=entity-tribe]').click(); cy.get('label[for=spend-yes]').click(); cy.get('label[for=us-yes]').click(); } diff --git a/backend/cypress/support/cross-validation.js b/backend/cypress/support/cross-validation.js index 0150841a30..75201b3161 100644 --- a/backend/cypress/support/cross-validation.js +++ b/backend/cypress/support/cross-validation.js @@ -1,5 +1,5 @@ export function testCrossValidation() { - cy.url().should('match', /\/audit\/cross-validation\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/cross-validation\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); // Cross val runs and passes cy.get(".usa-button").contains("Begin Validation").click(); @@ -7,9 +7,9 @@ export function testCrossValidation() { // Continue to the lock screen cy.get('.usa-button').contains('Proceed to certification').click(); - cy.url().should('match', /\/audit\/ready-for-certification\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/ready-for-certification\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); // Lock the submission for the certification steps cy.get('.usa-button').contains('Lock for certification').click(); - cy.url().should('match', /\/audit\/submission-progress\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); } diff --git a/backend/cypress/support/general-info.js b/backend/cypress/support/general-info.js index 8213ae7c9c..1f9eebbf31 100644 --- a/backend/cypress/support/general-info.js +++ b/backend/cypress/support/general-info.js @@ -10,7 +10,7 @@ export function testValidGeneralInfo() { cy.get('label[for=audit-period-annual]').click(); // Auditee information - cy.get('#auditee_name').type('Super Important name') + cy.get('#auditee_name').type('Audit McAuditee') cy.get('#ein').type('546000173'); cy.get('label[for=ein_not_an_ssn_attestation]').click(); cy.get('label[for=multiple-eins-yes]').click(); @@ -32,7 +32,7 @@ export function testValidGeneralInfo() { // Auditor information cy.get('#auditor_ein').type('987654321'); cy.get('label[for=auditor_ein_not_an_ssn_attestation]').click(); - cy.get('#auditor_firm_name').type('House of Audit'); + cy.get('#auditor_firm_name').type('House of Auditor'); // Pre-filled as USA // cy.get('#auditor_country').type('USA{enter}'); cy.get('#auditor_address_line_1').type('123 Around the corner'); @@ -50,5 +50,5 @@ export function testValidGeneralInfo() { cy.get('#continue').click(); - cy.url().should('match', /\/audit\/submission-progress\/[0-9A-Z]{17}$/); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); }; diff --git a/backend/cypress/support/tribal-audit-form.js b/backend/cypress/support/tribal-audit-form.js new file mode 100644 index 0000000000..d3f6137fbb --- /dev/null +++ b/backend/cypress/support/tribal-audit-form.js @@ -0,0 +1,16 @@ + +export function testTribalAuditPublic(){ + cy.get(`#is_tribal_information_authorized_to_be_public-yes`).click({force:true}); + cy.get('#tribal_authorization_certifying_official_name').type('John Wick'); + cy.get('#tribal_authorization_certifying_official_title').type('Offical'); + cy.get('#continue').click(); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}$/); +} + +export function testTribalAuditPrivate(){ + cy.get(`#is_tribal_information_authorized_to_be_public-no`).click({force:true}); + cy.get('#tribal_authorization_certifying_official_name').type('Clint Eastwood'); + cy.get('#tribal_authorization_certifying_official_title').type('Official'); + cy.get('#continue').click(); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}$/); +} \ No newline at end of file diff --git a/backend/cypress/support/workbook-uploads.js b/backend/cypress/support/workbook-uploads.js index 6a39e53c4a..7233321e27 100644 --- a/backend/cypress/support/workbook-uploads.js +++ b/backend/cypress/support/workbook-uploads.js @@ -22,7 +22,7 @@ function testWorkbookUpload(interceptUrl, uploadSelector, filename, will_interce ); cy.get('#continue').click(); - cy.url().should('match', /\/audit\/submission-progress\/[0-9A-Z]{17}/); + cy.url().should('match', /\/audit\/submission-progress\/[0-9]{4}-[0-9]{2}-GSAFAC-[0-9]{10}/); } export function testWorkbookFederalAwards(will_intercept = true) { diff --git a/backend/dissemination/README.md b/backend/dissemination/README.md index 0960f49c36..9907a7d974 100644 --- a/backend/dissemination/README.md +++ b/backend/dissemination/README.md @@ -1,6 +1,6 @@ # Deploying a new API -An API in PostgREST needs a few things to happen. +An API in PostgREST needs a few things to happen. 1. A JWT secret needs to be loaded into the PostgREST environment. 2. We need to tear down what was @@ -46,12 +46,12 @@ For symmetric use, that passphrase must be loaded into a GH Secret, and that sec Our JWT only lives at api.data.gov. We will put it in the `Authorization: Bearer ` header. In this way, only API requests that come through api.data.gov (meaning requests that go to api.fac.gov) will be executed by PostgREST. All other queries, from all other sources, will be rejected. -It is important that the role you choose matches the role we expect for public queries. Our schemas are attached to the role `api_fac_gov`. +It is important that the role you choose matches the role we expect for public queries. Our schemas are attached to the role `api_fac_gov`. For example: ``` -curl -X GET -H "Authorization: Bearer ${JWT}" "${API_FAC_URL}/general?limit=1" +curl -X GET -H "Authorization: Bearer ${JWT}" "${API_FAC_URL}/general?limit=1" ``` should return one item from the general view. API_FAC_URL might be `http://localhost:3000` in testing locally, or `https://api.fac.gov` when working live. @@ -72,7 +72,7 @@ checks this header, and if the correct role is present (`fac_gov_tribal_data_acc ## Standing up / tearing down -With each deployment of the stack, we should tear down and stand up the entire API. +With each deployment of the stack, we should tear down and stand up the entire API. 1. `fac drop_deprecated_schema_and_views` will tear down any deprecated APIs. Always run it. 1. `fac drop_api_schema` will tear down the active schema and everything associated with it. @@ -85,11 +85,12 @@ In other words: the API should always be stood up from a "blank slate" in the na # API versions -When adding a new API version. +When adding a new API version: -1. Create a folder in api/dissemination for the version name. E.g. `v1_0_1`. -2. Copy the contents of an existing API as a starting point. -3. Update `docker-compose.yml` and `docker-compose-web.yml` to change the `PGRST_DB_SCHEMAS` key to reflect all the active schemas. - 1. ADD TO THE END OF THIS LIST. The first entry is the default. Only add to the front of the list if we are certain the schema should become the new default. - 2. This is likely true of TESTED patch version bumps (v1_0_0 to v1_0_1), and *maybe* minor version bumps (v1_0_0 to v1_1_0). MAJOR bumps require change management messaging. -4. Update `APIViewTests` to make sure you're testing the right schema. (That file might want some love...) +1. Create a copy of an existing API directory within `FAC/backend/dissemination/api` and name it with your version bump of choice. + - For all files within this directory, replace all instances of the old API version with your new version. +2. Update `terraform/shared/modules/env/postgrest.tf` to use the new API version. +3. Update `docker-compose.yml` and `docker-compose-web.yml`: + - Change the values of `PGRST_DB_SCHEMAS` to your new API version. If previous versions of the API are needed, make the value a comma separated list and append your version to it. The first entry is the default, so only add to the front of the list if we are certain the schema should become the new default. See details on this [here](https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas) + - This is likely true of TESTED patch version bumps (v1_0_0 to v1_0_1), and *maybe* minor version bumps (v1_0_0 to v1_1_0). MAJOR bumps require change management messaging. +4. If previous versions of the API are needed, `APIViewTests` will need to be updated. At the time of writing this, it only tests the default API. diff --git a/backend/dissemination/api/api_v1_0_1/base.sql b/backend/dissemination/api/api_v1_0_1/base.sql new file mode 100644 index 0000000000..dedabe0cb7 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/base.sql @@ -0,0 +1,29 @@ +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'authenticator') THEN + RAISE NOTICE 'Role "authenticator" already exists. Skipping.'; + ELSE + CREATE ROLE authenticator LOGIN NOINHERIT NOCREATEDB NOCREATEROLE NOSUPERUSER; + END IF; +END +$do$; + +DO +$do$ +BEGIN + IF EXISTS ( + SELECT FROM pg_catalog.pg_roles + WHERE rolname = 'api_fac_gov') THEN + RAISE NOTICE 'Role "api_fac_gov" already exists. Skipping.'; + ELSE + CREATE ROLE api_fac_gov NOLOGIN; + END IF; +END +$do$; + +GRANT api_fac_gov TO authenticator; + +NOTIFY pgrst, 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_1/create_functions.sql b/backend/dissemination/api/api_v1_0_1/create_functions.sql new file mode 100644 index 0000000000..a5c340ffab --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/create_functions.sql @@ -0,0 +1,60 @@ +-- WARNING +-- Under PostgreSQL 12, the functions below work. +-- Under PostgreSQL 14, these will break. +-- +-- Note the differences: +-- +-- raise info 'Works under PostgreSQL 12'; +-- raise info 'request.header.x-magic %', (SELECT current_setting('request.header.x-magic', true)); +-- raise info 'request.jwt.claim.expires %', (SELECT current_setting('request.jwt.claim.expires', true)); +-- raise info 'Works under PostgreSQL 14'; +-- raise info 'request.headers::json->>x-magic %', (SELECT current_setting('request.headers', true)::json->>'x-magic'); +-- raise info 'request.jwt.claims::json->expires %', (SELECT current_setting('request.jwt.claims', true)::json->>'expires'); +-- +-- To quote the work of Dav Pilkey, "remember this now." + +create or replace function getter(base text, item text) returns text +as $getter$ +begin + return current_setting(concat(base, '.', item), true); +end; +$getter$ language plpgsql; + +create or replace function get_jwt_claim(item text) returns text +as $get_jwt_claim$ +begin + return getter('request.jwt.claim', item); +end; +$get_jwt_claim$ language plpgsql; + +create or replace function get_header(item text) returns text +as $get_header$ +begin + raise info 'request.header % %', item, getter('request.header', item); + return getter('request.header', item); +end; +$get_header$ LANGUAGE plpgsql; + +-- https://api-umbrella.readthedocs.io/en/latest/admin/api-backends/http-headers.html +-- I'd like to go to a model where we provide the API keys. +-- However, for now, we're going to look for a role attached to an api.data.gov account. +-- These come in on `X-Api-Roles` as a comma-separated string. +create or replace function has_tribal_data_access() returns boolean +as $has_tribal_data_access$ +declare + roles text; +begin + select get_header('x-api-roles') into roles; + return (roles like '%fac_gov_tribal_access%'); +end; +$has_tribal_data_access$ LANGUAGE plpgsql; + +create or replace function has_public_data_access_only() returns boolean +as $has_public_data_access_only$ +begin + return not has_tribal_data_access(); +end; +$has_public_data_access_only$ LANGUAGE plpgsql; + + +NOTIFY pgrst, 'reload schema'; \ No newline at end of file diff --git a/backend/dissemination/api/api_v1_0_1/create_schema.sql b/backend/dissemination/api/api_v1_0_1/create_schema.sql new file mode 100644 index 0000000000..28b2757db4 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/create_schema.sql @@ -0,0 +1,48 @@ +begin; + +do +$$ +begin + DROP SCHEMA IF EXISTS api_v1_0_1 CASCADE; + + if not exists (select schema_name from information_schema.schemata where schema_name = 'api_v1_0_1') then + create schema api_v1_0_1; + + -- Grant access to tables and views + alter default privileges + in schema api_v1_0_1 + grant select + -- this includes views + on tables + to api_fac_gov; + + -- Grant access to sequences, if we have them + grant usage on schema api_v1_0_1 to api_fac_gov; + grant select, usage on all sequences in schema api_v1_0_1 to api_fac_gov; + alter default privileges + in schema api_v1_0_1 + grant select, usage + on sequences + to api_fac_gov; + end if; +end +$$ +; + +-- This is the description +COMMENT ON SCHEMA api_v1_0_1 IS + 'The FAC dissemation API version 1.0.1.' +; + +-- https://postgrest.org/en/stable/references/api/openapi.html +-- This is the title +COMMENT ON SCHEMA api_v1_0_1 IS +$$v1.0.1 + +A RESTful API that serves data from the SF-SAC.$$; + +commit; + +notify pgrst, + 'reload schema'; + diff --git a/backend/dissemination/api/api_v1_0_1/create_views.sql b/backend/dissemination/api/api_v1_0_1/create_views.sql new file mode 100644 index 0000000000..808aed62de --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/create_views.sql @@ -0,0 +1,315 @@ + +begin; + +--------------------------------------- +-- finding_text +--------------------------------------- +create view api_v1_0_1.findings_text as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + ft.finding_ref_number, + ft.contains_chart_or_table, + ft.finding_text + from + dissemination_findingtext ft, + dissemination_general gen + where + (ft.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ft.id +; + +--------------------------------------- +-- additional_ueis +--------------------------------------- +create view api_v1_0_1.additional_ueis as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + uei.additional_uei + from + dissemination_general gen, + dissemination_additionaluei uei + where + (gen.report_id = uei.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by uei.id +; + +--------------------------------------- +-- finding +--------------------------------------- +create view api_v1_0_1.findings as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + finding.award_reference, + finding.reference_number, + finding.is_material_weakness, + finding.is_modified_opinion, + finding.is_other_findings, + finding.is_other_matters, + finding.prior_finding_ref_numbers, + finding.is_questioned_costs, + finding.is_repeat_finding, + finding.is_significant_deficiency, + finding.type_requirement + from + dissemination_finding finding, + dissemination_general gen + where + (finding.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by finding.id +; + +--------------------------------------- +-- federal award +--------------------------------------- +create view api_v1_0_1.federal_awards as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + award.award_reference, + award.federal_agency_prefix, + award.federal_award_extension, + award.additional_award_identification, + award.federal_program_name, + award.amount_expended, + award.cluster_name, + award.other_cluster_name, + award.state_cluster_name, + award.cluster_total, + award.federal_program_total, + award.is_major, + award.is_loan, + award.loan_balance, + award.is_direct, + award.audit_report_type, + award.findings_count, + award.is_passthrough_award, + award.passthrough_amount + from + dissemination_federalaward award, + dissemination_general gen + where + (award.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by award.id +; + + +--------------------------------------- +-- corrective_action_plan +--------------------------------------- +create view api_v1_0_1.corrective_action_plans as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ct.finding_ref_number, + ct.contains_chart_or_table, + ct.planned_action + from + dissemination_CAPText ct, + dissemination_General gen + where + (ct.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ct.id +; + +--------------------------------------- +-- notes_to_sefa +--------------------------------------- +create view api_v1_0_1.notes_to_sefa as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + note.note_title as title, + note.accounting_policies, + note.is_minimis_rate_used, + note.rate_explained, + note.content, + note.contains_chart_or_table + from + dissemination_general gen, + dissemination_note note + where + (note.report_id = gen.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by note.id +; + +--------------------------------------- +-- passthrough +--------------------------------------- +create view api_v1_0_1.passthrough as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + pass.award_reference, + pass.passthrough_id, + pass.passthrough_name + from + dissemination_general as gen, + dissemination_passthrough as pass + where + (gen.report_id = pass.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by pass.id +; + + +--------------------------------------- +-- general +--------------------------------------- +create view api_v1_0_1.general as + select + -- every table starts with report_id, UEI, and year + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + gen.auditee_certify_name, + gen.auditee_certify_title, + gen.auditee_contact_name, + gen.auditee_email, + gen.auditee_name, + gen.auditee_phone, + gen.auditee_contact_title, + gen.auditee_address_line_1, + gen.auditee_city, + gen.auditee_state, + gen.auditee_ein, + gen.auditee_zip, + -- auditor + gen.auditor_phone, + gen.auditor_state, + gen.auditor_city, + gen.auditor_contact_title, + gen.auditor_address_line_1, + gen.auditor_zip, + gen.auditor_country, + gen.auditor_contact_name, + gen.auditor_email, + gen.auditor_firm_name, + gen.auditor_foreign_address, + gen.auditor_ein, + -- agency + gen.cognizant_agency, + gen.oversight_agency, + -- dates + gen.date_created, + gen.ready_for_certification_date, + gen.auditor_certified_date, + gen.auditee_certified_date, + gen.submitted_date, + gen.fac_accepted_date, + gen.fy_end_date, + gen.fy_start_date, + gen.audit_type, + gen.gaap_results, + gen.sp_framework_basis, + gen.is_sp_framework_required, + gen.sp_framework_opinions, + gen.is_going_concern_included, + gen.is_internal_control_deficiency_disclosed, + gen.is_internal_control_material_weakness_disclosed, + gen.is_material_noncompliance_disclosed, + gen.dollar_threshold, + gen.is_low_risk_auditee, + gen.agencies_with_prior_findings, + gen.entity_type, + gen.number_months, + gen.audit_period_covered, + gen.total_amount_expended, + gen.type_audit_code, + gen.is_public, + gen.data_source + from + dissemination_General gen + where + (gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by gen.id +; + +--------------------------------------- +-- auditor (secondary auditor) +--------------------------------------- +create view api_v1_0_1.secondary_auditors as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + sa.auditor_ein, + sa.auditor_name, + sa.contact_name, + sa.contact_email, + sa.contact_phone, + sa.address_street, + sa.address_city, + sa.address_state, + sa.address_zipcode + from + dissemination_General gen, + dissemination_SecondaryAuditor sa + where + (sa.report_id = gen.report_id + and + gen.is_public=True) + or (gen.is_public=false and has_tribal_data_access()) + order by sa.id +; + +create view api_v1_0_1.additional_eins as + select + gen.report_id, + gen.auditee_uei, + gen.audit_year, + --- + ein.additional_ein + from + dissemination_general gen, + dissemination_additionalein ein + where + (gen.report_id = ein.report_id + and + gen.is_public = true) + or (gen.is_public = false and has_tribal_data_access()) + order by ein.id +; + + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_1/drop_schema.sql b/backend/dissemination/api/api_v1_0_1/drop_schema.sql new file mode 100644 index 0000000000..b182fd0600 --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/drop_schema.sql @@ -0,0 +1,11 @@ + +begin; + +DROP SCHEMA IF EXISTS api_v1_0_1 CASCADE; +-- DROP ROLE IF EXISTS authenticator; +-- DROP ROLE IF EXISTS api_fac_gov; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api/api_v1_0_1/drop_views.sql b/backend/dissemination/api/api_v1_0_1/drop_views.sql new file mode 100644 index 0000000000..b962190fec --- /dev/null +++ b/backend/dissemination/api/api_v1_0_1/drop_views.sql @@ -0,0 +1,15 @@ +begin; + + drop table if exists api_v1_0_1.metadata; + drop view if exists api_v1_0_1.general; + drop view if exists api_v1_0_1.auditor; + drop view if exists api_v1_0_1.federal_award; + drop view if exists api_v1_0_1.finding; + drop view if exists api_v1_0_1.finding_text; + drop view if exists api_v1_0_1.cap_text; + drop view if exists api_v1_0_1.note; + +commit; + +notify pgrst, + 'reload schema'; diff --git a/backend/dissemination/api_versions.py b/backend/dissemination/api_versions.py index 8523e9c0c5..1ee8f7d0b4 100644 --- a/backend/dissemination/api_versions.py +++ b/backend/dissemination/api_versions.py @@ -5,7 +5,7 @@ # These are API versions we want live. live = [ # These are API versions we have in flight. - "api_v1_0_0", + "api_v1_0_1", ] # These are API versions we have deprecated. diff --git a/backend/dissemination/migrations/0002_general_fac_accepted_date.py b/backend/dissemination/migrations/0002_general_fac_accepted_date.py new file mode 100644 index 0000000000..81eea6ad35 --- /dev/null +++ b/backend/dissemination/migrations/0002_general_fac_accepted_date.py @@ -0,0 +1,30 @@ +# Generated by Django 4.2.5 on 2023-09-29 17:06 + +from django.db import migrations, models + + +# Copies the values of submitted_date to fac_accepted_date +def copy_submitted_to_accepted(apps, schema_editor): + General = apps.get_model("dissemination", "General") + for report in General.objects.all(): + if not report.fac_accepted_date: + report.fac_accepted_date = report.submitted_date + report.save() + + +class Migration(migrations.Migration): + dependencies = [ + ("dissemination", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="general", + name="fac_accepted_date", + field=models.DateField( + null=True, + verbose_name="The date at which the audit transitioned to 'accepted'", + ), + ), + migrations.RunPython(copy_submitted_to_accepted), + ] diff --git a/backend/dissemination/migrations/0003_alter_general_fac_accepted_date.py b/backend/dissemination/migrations/0003_alter_general_fac_accepted_date.py new file mode 100644 index 0000000000..5cfc4b0c68 --- /dev/null +++ b/backend/dissemination/migrations/0003_alter_general_fac_accepted_date.py @@ -0,0 +1,20 @@ +# Generated by Django 4.2.5 on 2023-09-29 17:22 +# This migration removes null=True for fac_accepted_date + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("dissemination", "0002_general_fac_accepted_date"), + ] + + operations = [ + migrations.AlterField( + model_name="general", + name="fac_accepted_date", + field=models.DateField( + verbose_name="The date at which the audit transitioned to 'accepted'" + ), + ), + ] diff --git a/backend/dissemination/models.py b/backend/dissemination/models.py index 01899ef306..d8cd7aa7dc 100644 --- a/backend/dissemination/models.py +++ b/backend/dissemination/models.py @@ -345,6 +345,9 @@ class General(models.Model): submitted_date = models.DateField( "The date at which the audit transitioned to 'submitted'", ) + fac_accepted_date = models.DateField( + "The date at which the audit transitioned to 'accepted'", + ) # auditor_signature_date = models.DateField( # "The date on which the auditor signed the audit", # ) diff --git a/backend/dissemination/tests.py b/backend/dissemination/tests.py index 5af4b3eeab..e1ff36cd3e 100644 --- a/backend/dissemination/tests.py +++ b/backend/dissemination/tests.py @@ -9,8 +9,6 @@ from config import settings -api_schemas = ["api_v1_0_0"] - class APIViewTests(TestCase): def setUp(self): diff --git a/backend/docker-compose-web.yml b/backend/docker-compose-web.yml index 1e5914ff31..0cccfcfe66 100644 --- a/backend/docker-compose-web.yml +++ b/backend/docker-compose-web.yml @@ -74,12 +74,13 @@ services: PGRST_DB_URI: postgres://postgres@db:5432/postgres PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon - PGRST_DB_SCHEMAS: "api_v1_0_0" + # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas + PGRST_DB_SCHEMAS: "api_v1_0_1" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments depends_on: db: condition: service_healthy - + volumes: postgres-data: minio-vol: diff --git a/backend/docker-compose.yml b/backend/docker-compose.yml index f2822557af..ff337c31a6 100644 --- a/backend/docker-compose.yml +++ b/backend/docker-compose.yml @@ -108,7 +108,8 @@ services: PGRST_DB_URI: postgres://postgres@db:5432/postgres PGRST_OPENAPI_SERVER_PROXY_URI: http://127.0.0.1:3000 PGRST_DB_ANON_ROLE: anon - PGRST_DB_SCHEMAS: "api_v1_0_0" + # See https://postgrest.org/en/stable/references/api/schemas.html#multiple-schemas for multiple schemas + PGRST_DB_SCHEMAS: "api_v1_0_1" PGRST_JWT_SECRET: ${PGRST_JWT_SECRET:-32_chars_fallback_secret_testing} # Fallback value for testing environments depends_on: db: diff --git a/backend/schemas/output/excel/json/federal-awards-workbook.json b/backend/schemas/output/excel/json/federal-awards-workbook.json index e1d31b392c..6f3771849c 100644 --- a/backend/schemas/output/excel/json/federal-awards-workbook.json +++ b/backend/schemas/output/excel/json/federal-awards-workbook.json @@ -89,8 +89,8 @@ "format": "dollar", "formula": "=SUM('Form'!F$FIRSTROW:F$LASTROW)", "help": { - "link": "https://fac.gov/documentation/validation/#positive_number", - "text": "The number must be zero or greater" + "link": "https://fac.gov/documentation/validation/#any_number", + "text": "Must be a number" }, "keep_locked": true, "range_cell": "B5", @@ -99,9 +99,9 @@ "title_cell": "A5", "type": "single_cell", "validation": { - "custom_error": "This cell must be a positive number", - "custom_title": "Positive numbers", - "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))", + "custom_error": "This cell must be a number", + "custom_title": "Numbers", + "formula1": "=ISNUMBER(FIRSTCELLREF)", "type": "custom" }, "width": 36 @@ -197,17 +197,17 @@ { "format": "dollar", "help": { - "link": "https://fac.gov/documentation/validation/#positive_number", - "text": "The number must be zero or greater" + "link": "https://fac.gov/documentation/validation/#any_number", + "text": "Must be a number" }, "range_name": "amount_expended", "title": "Amount Expended", "title_cell": "F1", "type": "open_range", "validation": { - "custom_error": "This cell must be a positive number", - "custom_title": "Positive numbers", - "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))", + "custom_error": "This cell must be a number", + "custom_title": "Numbers", + "formula1": "=ISNUMBER(FIRSTCELLREF)", "type": "custom" } }, @@ -259,8 +259,8 @@ "format": "dollar", "formula": "=SUMIFS(amount_expended,cfda_key,V{0})", "help": { - "link": "https://fac.gov/documentation/validation/#positive_number", - "text": "The number must be zero or greater" + "link": "https://fac.gov/documentation/validation/#any_number", + "text": "Must be a number" }, "keep_locked": true, "range_name": "federal_program_total", @@ -268,9 +268,9 @@ "title_cell": "J1", "type": "open_range", "validation": { - "custom_error": "This cell must be a positive number", - "custom_title": "Positive numbers", - "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))", + "custom_error": "This cell must be a number", + "custom_title": "Numbers", + "formula1": "=ISNUMBER(FIRSTCELLREF)", "type": "custom" } }, @@ -278,8 +278,8 @@ "format": "dollar", "formula": "=IF(G{0}=\"OTHER CLUSTER NOT LISTED ABOVE\",SUMIFS(amount_expended,uniform_other_cluster_name,X{0}), IF(AND(OR(G{0}=\"N/A\",G{0}=\"\"),H{0}=\"\"),0,IF(G{0}=\"STATE CLUSTER\",SUMIFS(amount_expended,uniform_state_cluster_name,W{0}),SUMIFS(amount_expended,cluster_name,G{0}))))", "help": { - "link": "https://fac.gov/documentation/validation/#positive_number", - "text": "The number must be zero or greater" + "link": "https://fac.gov/documentation/validation/#any_number", + "text": "Must be a number" }, "keep_locked": true, "range_name": "cluster_total", @@ -287,9 +287,9 @@ "title_cell": "K1", "type": "open_range", "validation": { - "custom_error": "This cell must be a positive number", - "custom_title": "Positive numbers", - "formula1": "=AND(ISNUMBER(FIRSTCELLREF),OR(SIGN(FIRSTCELLREF)=0,SIGN(FIRSTCELLREF)=1))", + "custom_error": "This cell must be a number", + "custom_title": "Numbers", + "formula1": "=ISNUMBER(FIRSTCELLREF)", "type": "custom" } }, @@ -393,8 +393,8 @@ { "format": "dollar", "help": { - "link": "https://fac.gov/documentation/validation/#positive_number", - "text": "The number must be zero or greater" + "link": "https://fac.gov/documentation/validation/#any_number", + "text": "Must be a number" }, "range_name": "subrecipient_amount", "title": "If yes (Passed Through), Amount Passed Through to Subrecipients", diff --git a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx index 4e6300ed6e..65f0706476 100644 Binary files a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx and b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx index b364f222bb..06b64d8224 100644 Binary files a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx and b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx index 657d46c20f..1414850e05 100644 Binary files a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx and b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx index 99b4a93e66..93c1af0e2f 100644 Binary files a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx and b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx index 760db8953d..f24b604269 100644 Binary files a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx and b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx index 19dff6842a..3606675460 100644 Binary files a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx and b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx index 10769e1016..fa579bb030 100644 Binary files a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx and b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx index 590844f688..dad17d41c0 100644 Binary files a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx and b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx differ diff --git a/backend/schemas/output/sections/FederalAwards.schema.json b/backend/schemas/output/sections/FederalAwards.schema.json index 8df5f0fafd..0736da14f6 100644 --- a/backend/schemas/output/sections/FederalAwards.schema.json +++ b/backend/schemas/output/sections/FederalAwards.schema.json @@ -37,6 +37,12 @@ }, "cluster": { "allOf": [ + { + "required": [ + "cluster_name", + "cluster_total" + ] + }, { "if": { "properties": { @@ -165,7 +171,6 @@ "type": "string" }, "cluster_total": { - "minimum": 0, "type": "number" }, "other_cluster_name": { @@ -175,15 +180,16 @@ "type": "string" } }, - "required": [ - "cluster_name", - "cluster_total" - ], "type": "object" }, "direct_or_indirect_award": { "additionalProperties": false, "allOf": [ + { + "required": [ + "is_direct" + ] + }, { "if": { "properties": { @@ -243,14 +249,16 @@ "type": "string" } }, - "required": [ - "is_direct" - ], "type": "object" }, "loan_or_loan_guarantee": { "additionalProperties": false, "allOf": [ + { + "required": [ + "is_guaranteed" + ] + }, { "if": { "properties": { @@ -307,14 +315,22 @@ ] } }, - "required": [ - "is_guaranteed" - ], "type": "object" }, "program": { "additionalProperties": false, "allOf": [ + { + "required": [ + "program_name", + "federal_agency_prefix", + "three_digit_extension", + "is_major", + "number_of_audit_findings", + "federal_program_total", + "amount_expended" + ] + }, { "properties": { "number_of_audit_findings": { @@ -380,7 +396,6 @@ "type": "string" }, "amount_expended": { - "minimum": 0, "type": "number" }, "audit_report_type": { @@ -407,7 +422,6 @@ "type": "string" }, "federal_program_total": { - "minimum": 0, "type": "number" }, "is_major": { @@ -428,20 +442,16 @@ "type": "string" } }, - "required": [ - "program_name", - "federal_agency_prefix", - "three_digit_extension", - "is_major", - "number_of_audit_findings", - "federal_program_total", - "amount_expended" - ], "type": "object" }, "subrecipients": { "additionalProperties": false, "allOf": [ + { + "required": [ + "is_passed" + ] + }, { "if": { "properties": { @@ -482,13 +492,9 @@ "type": "string" }, "subrecipient_amount": { - "minimum": 0, "type": "number" } }, - "required": [ - "is_passed" - ], "type": "object" } }, @@ -505,7 +511,6 @@ "type": "array" }, "total_amount_expended": { - "minimum": 0, "type": "number" } }, diff --git a/backend/schemas/output/sections/TribalAccess.schema.json b/backend/schemas/output/sections/TribalAccess.schema.json index 4ab0f59366..2918a854f6 100644 --- a/backend/schemas/output/sections/TribalAccess.schema.json +++ b/backend/schemas/output/sections/TribalAccess.schema.json @@ -21,5 +21,5 @@ ], "title": "TribalAccess", "type": "object", - "version": null + "version": 20230927 } diff --git a/backend/schemas/source/excel/libs/Help.libsonnet b/backend/schemas/source/excel/libs/Help.libsonnet index d9396171a6..e6700cd216 100644 --- a/backend/schemas/source/excel/libs/Help.libsonnet +++ b/backend/schemas/source/excel/libs/Help.libsonnet @@ -48,6 +48,10 @@ local make_url = function(anchor) text: 'The number must be zero or greater', link: make_url('positive_number'), }, + any_number: { + text: 'Must be a number', + link: make_url('any_number'), + }, prior_references: { text: 'Must be a comma-separated list of reference numbers (YYYY-NNN) or N/A.', link: make_url('prior_references'), diff --git a/backend/schemas/source/excel/libs/SheetValidations.libsonnet b/backend/schemas/source/excel/libs/SheetValidations.libsonnet index 32b7c24d89..61e735d1a2 100644 --- a/backend/schemas/source/excel/libs/SheetValidations.libsonnet +++ b/backend/schemas/source/excel/libs/SheetValidations.libsonnet @@ -52,6 +52,14 @@ local PositiveNumberValidation = { custom_title: 'Positive numbers', }; +local NumberValidation = { + type: 'custom', + // Is it a number ? + formula1: '=ISNUMBER(FIRSTCELLREF)', + custom_error: 'This cell must be a number', + custom_title: 'Numbers', +}; + local ReferenceNumberValidation = { type: 'custom', //It is neccessary to allow blank otherwise user cannot delete the value @@ -87,6 +95,7 @@ local AwardReferenceValidation = { NoValidation: { type: 'NOVALIDATION' }, FAPPrefixValidation: FAPPrefixValidation, PositiveNumberValidation: PositiveNumberValidation, + NumberValidation: NumberValidation, LookupValidation: LookupValidation, RangeLookupValidation: RangeLookupValidation, StringOfLengthNine: StringOfSize(9), diff --git a/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet b/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet index d376220c96..d50a6c6fba 100644 --- a/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet @@ -79,8 +79,8 @@ local single_cells = [ // FIXME MSHD: for improvement, will need to pull F from this formula and retrieve it dynamically. formula: "=SUM('" + awardSheet + "'!F$FIRSTROW:F$LASTROW)", width: 36, - help: Help.positive_number, - validation: SV.PositiveNumberValidation, + help: Help.any_number, + validation: SV.NumberValidation, }, ]; @@ -137,9 +137,9 @@ local open_ranges_defns = [ [ Sheets.open_range { format: 'dollar', - help: Help.positive_number, + help: Help.any_number, }, - SV.PositiveNumberValidation, + SV.NumberValidation, 'Amount Expended', amountExpendedNamedRange, ], @@ -176,9 +176,9 @@ local open_ranges_defns = [ keep_locked: true, format: 'dollar', formula: '=SUMIFS(' + amountExpendedNamedRange + ',' + cfdaKeyNamedRange + ',V{0})', - help: Help.positive_number, + help: Help.any_number, }, - SV.PositiveNumberValidation, + SV.NumberValidation, 'Federal Program Total', 'federal_program_total', ], @@ -187,9 +187,9 @@ local open_ranges_defns = [ keep_locked: true, format: 'dollar', formula: '=IF(G{0}="' + Base.Const.OTHER_CLUSTER + '",SUMIFS(' + amountExpendedNamedRange + ',' + uniformOtherClusterNamedRange + ',X{0}), IF(AND(OR(G{0}="' + Base.Const.NA + '",G{0}=""),H{0}=""),0,IF(G{0}="' + Base.Const.STATE_CLUSTER + '",SUMIFS(' + amountExpendedNamedRange + ',' + uniformStateClusterNamedRange + ',W{0}),SUMIFS(' + amountExpendedNamedRange + ',' + clusterNamedRange + ',G{0}))))', - help: Help.positive_number, + help: Help.any_number, }, - SV.PositiveNumberValidation, + SV.NumberValidation, 'Cluster Total', 'cluster_total', ], @@ -248,7 +248,7 @@ local open_ranges_defns = [ [ Sheets.open_range { format: 'dollar', - help: Help.positive_number, + help: Help.any_number, }, SV.NoValidation, 'If yes (Passed Through), Amount Passed Through to Subrecipients', diff --git a/backend/schemas/source/sections/FederalAwards.schema.jsonnet b/backend/schemas/source/sections/FederalAwards.schema.jsonnet index db45c5536d..120b8c60ac 100644 --- a/backend/schemas/source/sections/FederalAwards.schema.jsonnet +++ b/backend/schemas/source/sections/FederalAwards.schema.jsonnet @@ -14,6 +14,9 @@ local Validations = { required: ['passthrough_name'], }, LoanOrLoanGuaranteeValidations: [ + { + required: ['is_guaranteed'], + }, { 'if': { properties: { @@ -42,6 +45,9 @@ local Validations = { }, ], SubrecipientValidations: [ + { + required: ['is_passed'], + }, { 'if': { properties: { @@ -72,6 +78,17 @@ local Validations = { }, ], ProgramValidations: [ + { + required: [ + 'program_name', + 'federal_agency_prefix', + 'three_digit_extension', + 'is_major', + 'number_of_audit_findings', + 'federal_program_total', + 'amount_expended', + ], + }, { properties: { number_of_audit_findings: Types.integer { @@ -152,12 +169,12 @@ local Parts = { cluster_name: Types.string, other_cluster_name: Types.string, state_cluster_name: Types.string, - cluster_total: Types.number { - minimum: 0, - }, + cluster_total: Types.number, }, - required: ['cluster_name', 'cluster_total'], allOf: [ + { + required: ['cluster_name', 'cluster_total'], + }, { 'if': { properties: { @@ -239,8 +256,10 @@ local Parts = { items: Validations.PassThroughEntity, }, }, - required: ['is_direct'], allOf: [ + { + required: ['is_direct'], + }, { 'if': { properties: { @@ -284,20 +303,14 @@ local Parts = { ], }, }, - required: [ - 'is_guaranteed', - ], allOf: Validations.LoanOrLoanGuaranteeValidations, }, Subrecipients: Types.object { additionalProperties: false, properties: { is_passed: Base.Enum.YorN, - subrecipient_amount: Types.number { - minimum: 0, - }, + subrecipient_amount: Types.number, }, - required: ['is_passed'], allOf: Validations.SubrecipientValidations, }, Program: Types.object { @@ -307,25 +320,12 @@ local Parts = { three_digit_extension: Base.Compound.ThreeDigitExtension, additional_award_identification: Types.string, program_name: Types.string, - amount_expended: Types.number { - minimum: 0, - }, - federal_program_total: Types.number { - minimum: 0, - }, + amount_expended: Types.number, + federal_program_total: Types.number, is_major: Base.Enum.YorN, audit_report_type: Base.Enum.MajorProgramAuditReportType, number_of_audit_findings: Types.integer, }, - required: [ - 'program_name', - 'federal_agency_prefix', - 'three_digit_extension', - 'is_major', - 'number_of_audit_findings', - 'federal_program_total', - 'amount_expended', - ], allOf: Validations.ProgramValidations, }, }; @@ -375,9 +375,7 @@ local FederalAwards = Types.object { federal_awards: Types.array { items: FederalAwardEntry, }, - total_amount_expended: Types.number { - minimum: 0, - }, + total_amount_expended: Types.number, }, required: ['auditee_uei', 'total_amount_expended'], title: 'FederalAward', diff --git a/backend/schemas/source/sections/GeneralInformation.schema.jsonnet b/backend/schemas/source/sections/GeneralInformation.schema.jsonnet index 538d4a1b53..e783a483de 100644 --- a/backend/schemas/source/sections/GeneralInformation.schema.jsonnet +++ b/backend/schemas/source/sections/GeneralInformation.schema.jsonnet @@ -18,7 +18,7 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check format: 'date', }, Base.Compound.EmptyString, - ] + ], }, auditee_fiscal_period_end: Types.string { oneOf: [ @@ -26,19 +26,19 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check format: 'date', }, Base.Compound.EmptyString, - ] + ], }, audit_type: { oneOf: [ Base.Enum.AuditType, Base.Compound.EmptyString, - ] + ], }, audit_period_covered: { oneOf: [ Base.Enum.AuditPeriod, Base.Compound.EmptyString, - ] + ], }, audit_period_other_months: Types.string { maxLength: 100, @@ -68,7 +68,7 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check title: 'State', }, Base.Compound.EmptyString, - ] + ], }, auditee_zip: { anyOf: [ @@ -87,7 +87,7 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check oneOf: [ Base.Compound.UnitedStatesPhone, Base.Compound.EmptyString, - ] + ], }, auditee_email: Types.string { oneOf: [ @@ -95,11 +95,11 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check format: 'email', }, Base.Compound.EmptyString, - ] + ], }, // Auditor information - auditor_ein: { + auditor_ein: { oneOf: [ Base.Compound.EmployerIdentificationNumber, Base.Compound.EmptyString, @@ -144,14 +144,14 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check oneOf: [ Base.Compound.UnitedStatesPhone, Base.Compound.EmptyString, - ] + ], }, auditor_email: Types.string { oneOf: [ { format: 'email', }, - Base.Compound.EmptyString + Base.Compound.EmptyString, ], maxLength: 100, }, @@ -231,7 +231,7 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check Base.Enum.UnitedStatesStateAbbr, Base.Compound.EmptyString, ], - } + }, }, }, }, diff --git a/backend/schemas/source/sections/GeneralInformationComplete.schema.jsonnet b/backend/schemas/source/sections/GeneralInformationComplete.schema.jsonnet index 8995183c93..407eaac153 100644 --- a/backend/schemas/source/sections/GeneralInformationComplete.schema.jsonnet +++ b/backend/schemas/source/sections/GeneralInformationComplete.schema.jsonnet @@ -107,7 +107,7 @@ Requires most fields, has consitional checks for conditional fields. minLength: 1, }, - // Others + // Others is_usa_based: Types.boolean, met_spending_threshold: Types.boolean, user_provided_organization_type: Base.Enum.OrganizationType, @@ -116,7 +116,7 @@ Requires most fields, has consitional checks for conditional fields. secondary_auditors_exist: Types.boolean, }, allOf: [ - // If audit_period_covered is 'other', then audit_period_other_months should + // If audit_period_covered is 'other', then audit_period_other_months should // have a value. Otherwise, it should have no value. { anyOf: [ @@ -205,7 +205,7 @@ Requires most fields, has consitional checks for conditional fields. const: true, }, }, - } + }, ], required: [ 'audit_type', @@ -236,7 +236,7 @@ Requires most fields, has consitional checks for conditional fields. 'multiple_eins_covered', 'multiple_ueis_covered', 'secondary_auditors_exist', - 'user_provided_organization_type' + 'user_provided_organization_type', ], title: 'GeneralInformation', type: 'object', diff --git a/backend/schemas/source/sections/TribalAccess.schema.jsonnet b/backend/schemas/source/sections/TribalAccess.schema.jsonnet index 26d113a90b..03f226a979 100644 --- a/backend/schemas/source/sections/TribalAccess.schema.jsonnet +++ b/backend/schemas/source/sections/TribalAccess.schema.jsonnet @@ -18,7 +18,7 @@ Typechecks fields, but allows for empty data as well. Contains conditional Check required: [ 'tribal_authorization_certifying_official_title', 'is_tribal_information_authorized_to_be_public', - 'tribal_authorization_certifying_official_name' + 'tribal_authorization_certifying_official_name', ], title: 'TribalAccess', type: 'object', diff --git a/terraform/shared/modules/env/postgrest.tf b/terraform/shared/modules/env/postgrest.tf index 390919796b..21e1891cfc 100644 --- a/terraform/shared/modules/env/postgrest.tf +++ b/terraform/shared/modules/env/postgrest.tf @@ -32,7 +32,7 @@ resource "cloudfoundry_app" "postgrest" { environment = { PGRST_DB_URI : cloudfoundry_service_key.postgrest.credentials.uri - PGRST_DB_SCHEMAS : "api_v1_0_0" + PGRST_DB_SCHEMAS : "api_v1_0_1" PGRST_DB_ANON_ROLE : "anon" PGRST_JWT_SECRET : var.pgrst_jwt_secret PGRST_DB_MAX_ROWS : 20000