diff --git a/.github/workflows/new-relic-deployment.yml b/.github/workflows/new-relic-deployment.yml new file mode 100644 index 0000000000..2a6e4049f6 --- /dev/null +++ b/.github/workflows/new-relic-deployment.yml @@ -0,0 +1,26 @@ +name: Record Deployment And Add New Relic Monitor +on: + push: + branches: + - main + - prod + tags: + - v1.* + +jobs: + newrelic: + runs-on: ubuntu-latest + name: New Relic Record Deployment + steps: + # This step builds a var with the release tag value to use later + - name: Set Release Version from Tag + run: echo "RELEASE_VERSION=${{ github.ref_name }}" >> $GITHUB_ENV + # This step creates a new Change Tracking Marker + - name: Add New Relic Application Deployment Marker + uses: newrelic/deployment-marker-action@v2.3.0 + with: + apiKey: ${{ secrets.NEW_RELIC_API_KEY }} + guid: ${{ secrets.NEW_RELIC_DEPLOYMENT_ENTITY_GUID }} + version: "${{ env.RELEASE_VERSION }}" + user: "${{ github.actor }}" + diff --git a/backend/.profile b/backend/.profile index fc2df6596a..fe09534e13 100644 --- a/backend/.profile +++ b/backend/.profile @@ -19,7 +19,10 @@ export no_proxy="${S3_ENDPOINT_FOR_NO_PROXY},${S3_FIPS_ENDPOINT_FOR_NO_PROXY},${ export NEW_RELIC_LICENSE_KEY="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "newrelic-creds" ".[][] | select(.name == \$service_name) | .credentials.NEW_RELIC_LICENSE_KEY")" # Set the application name for New Relic telemetry. -export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)" +# this did not work on preview, so, since gsa-fac-app works, keeping that there +#export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)-$(echo "$VCAP_APPLICATION" | jq -r .space_name)" +export NEW_RELIC_APP_NAME="$(echo "$VCAP_APPLICATION" | jq -r .application_name)-app" + # Set the environment name for New Relic telemetry. export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)" @@ -28,19 +31,20 @@ export NEW_RELIC_ENVIRONMENT="$(echo "$VCAP_APPLICATION" | jq -r .space_name)" export NEW_RELIC_LOG=stdout # Logging level, (critical, error, warning, info and debug). Default to info -export NEW_RELIC_LOG_LEVEL=error +export NEW_RELIC_LOG_LEVEL=info # https://docs.newrelic.com/docs/security/security-privacy/compliance/fedramp-compliant-endpoints/ export NEW_RELIC_HOST="gov-collector.newrelic.com" # https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy -https_proxy_protocol="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.protocol")" -https_proxy_domain="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.domain")" -https_proxy_port="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.port")" +#https_proxy_protocol="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.protocol")" +#https_proxy_domain="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.domain")" +#https_proxy_port="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.port")" -export NEW_RELIC_PROXY_HOST="$https_proxy_protocol://$https_proxy_domain" -export NEW_RELIC_PROXY_PORT="$https_proxy_port" -export NEW_RELIC_PROXY_USER="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.username")" -export NEW_RELIC_PROXY_PASS="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.password")" +#export NEW_RELIC_PROXY_SCHEME="$https_proxy_protocol" +#export NEW_RELIC_PROXY_HOST="$https_proxy_domain" +#export NEW_RELIC_PROXY_PORT="$https_proxy_port" +#export NEW_RELIC_PROXY_USER="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.username")" +#export NEW_RELIC_PROXY_PASS="$(echo "$VCAP_SERVICES" | jq --raw-output --arg service_name "https-proxy-creds" ".[][] | select(.name == \$service_name) | .credentials.password")" # We only want to run migrate and collecstatic for the first app instance, not # for additional app instances, so we gate all of this behind CF_INSTANCE_INDEX diff --git a/backend/audit/file_downloads.py b/backend/audit/file_downloads.py index c64668155f..e003d77212 100644 --- a/backend/audit/file_downloads.py +++ b/backend/audit/file_downloads.py @@ -2,7 +2,6 @@ from django.conf import settings from django.http import Http404 -from django.shortcuts import get_object_or_404 from boto3 import client as boto3_client from botocore.client import ClientError, Config @@ -14,11 +13,21 @@ def get_filename(sac, file_type): if file_type == "report": - file_obj = get_object_or_404(SingleAuditReportFile, sac=sac) - return f"singleauditreport/{file_obj.filename}" + try: + file_obj = SingleAuditReportFile.objects.filter(sac=sac).latest( + "date_created" + ) + return f"singleauditreport/{file_obj.filename}" + except SingleAuditReportFile.DoesNotExist: + raise Http404() else: - file_obj = get_object_or_404(ExcelFile, sac=sac, form_section=file_type) - return f"excel/{file_obj.filename}" + try: + file_obj = ExcelFile.objects.filter(sac=sac, form_section=file_type).latest( + "date_created" + ) + return f"excel/{file_obj.filename}" + except ExcelFile.DoesNotExist: + raise Http404() def file_exists(filename): diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has-bad-yes-or-no-inputs.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has-bad-yes-or-no-inputs.xlsx new file mode 100644 index 0000000000..ad2b636939 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has-bad-yes-or-no-inputs.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has_bad_prior_references.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has_bad_prior_references.xlsx new file mode 100644 index 0000000000..048081ceb5 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/has_bad_prior_references.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/missing-required-fields.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/missing-required-fields.xlsx new file mode 100644 index 0000000000..591fbf9285 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards-audit-findings/missing-required-fields.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/federal-awards-171944.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/federal-awards-171944.xlsx deleted file mode 100644 index 05738cf61a..0000000000 Binary files a/backend/audit/fixtures/workbooks/should_fail/federal-awards/federal-awards-171944.xlsx and /dev/null differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-yes-or-no-inputs.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-yes-or-no-inputs.xlsx new file mode 100644 index 0000000000..e3ee9b49ee Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-bad-yes-or-no-inputs.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx new file mode 100644 index 0000000000..871257262a Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/has-decimal-dollar-amounts.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_fail/federal-awards/missing_required_fields.xlsx b/backend/audit/fixtures/workbooks/should_fail/federal-awards/missing_required_fields.xlsx new file mode 100644 index 0000000000..297fbb17c2 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_fail/federal-awards/missing_required_fields.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-eins-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-eins-100010.xlsx index e6668940bc..71df332bee 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-eins-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-eins-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-ueis-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-ueis-100010.xlsx index 84402986c4..48778e1dbd 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-ueis-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/additional-ueis-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-100010.xlsx index a1b42a4b51..02e005c0e3 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-text-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-text-100010.xlsx index 8033d7fa6e..4f11671ccc 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-text-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/audit-findings-text-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/corrective-action-plan-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/corrective-action-plan-100010.xlsx index cc8e50da57..cc74c0e49b 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/corrective-action-plan-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/corrective-action-plan-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/federal-awards-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/federal-awards-100010.xlsx index 021eaeacf5..61bfa18a4a 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/federal-awards-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/federal-awards-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/notes-to-sefa-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/notes-to-sefa-100010.xlsx index 56b0be3276..1574fd5879 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/notes-to-sefa-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/notes-to-sefa-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/100010-22/secondary-auditors-100010.xlsx b/backend/audit/fixtures/workbooks/should_pass/100010-22/secondary-auditors-100010.xlsx index 2da594e83b..2e941c9320 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/100010-22/secondary-auditors-100010.xlsx and b/backend/audit/fixtures/workbooks/should_pass/100010-22/secondary-auditors-100010.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-eins-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-eins-134732.xlsx index 400036e50f..cfaf02d8e3 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-eins-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-eins-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-ueis-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-ueis-134732.xlsx index 329b012a0c..e6c1dca815 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-ueis-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/additional-ueis-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-134732.xlsx index 17e397981b..604a04843e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-text-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-text-134732.xlsx index d601552ecb..0c973a5a79 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-text-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/audit-findings-text-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/corrective-action-plan-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/corrective-action-plan-134732.xlsx index 093956b8f1..5e207d61b3 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/corrective-action-plan-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/corrective-action-plan-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/federal-awards-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/federal-awards-134732.xlsx index b1d50e36d4..1dbea9a55e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/federal-awards-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/federal-awards-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/notes-to-sefa-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/notes-to-sefa-134732.xlsx index 78acbd2f09..6aac37765e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/notes-to-sefa-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/notes-to-sefa-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/secondary-auditors-134732.xlsx b/backend/audit/fixtures/workbooks/should_pass/134732-21/secondary-auditors-134732.xlsx index 2772292de3..bc77a53f93 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/134732-21/secondary-auditors-134732.xlsx and b/backend/audit/fixtures/workbooks/should_pass/134732-21/secondary-auditors-134732.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/134732-21/test-array-134732.json b/backend/audit/fixtures/workbooks/should_pass/134732-21/test-array-134732.json index d2a4636546..004b2d2279 100644 --- a/backend/audit/fixtures/workbooks/should_pass/134732-21/test-array-134732.json +++ b/backend/audit/fixtures/workbooks/should_pass/134732-21/test-array-134732.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" @@ -14,7 +14,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [ { "fields": [ @@ -59,7 +59,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [ { "fields": [ @@ -1303,7 +1303,7 @@ }, { "endpoint": "findings_text", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [ { "fields": [ @@ -1348,7 +1348,7 @@ }, { "endpoint": "findings", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [ { "fields": [ @@ -1459,7 +1459,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [ { "fields": [ @@ -1491,7 +1491,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2021TEST000134732", + "report_id": "2021-12-TSTDAT-0000134732", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-eins-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-eins-147110.xlsx index 0d66b281f9..bf62c6f045 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-eins-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-eins-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-ueis-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-ueis-147110.xlsx index 8166742487..a80f61c24c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-ueis-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/additional-ueis-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-147110.xlsx index 74ec6ac966..d562f02378 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-text-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-text-147110.xlsx index 315578d335..031a2e7f6c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-text-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/audit-findings-text-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/corrective-action-plan-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/corrective-action-plan-147110.xlsx index 8e310b4ee4..b09b02b095 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/corrective-action-plan-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/corrective-action-plan-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/federal-awards-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/federal-awards-147110.xlsx index 09bd30dccf..86df269a1f 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/federal-awards-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/federal-awards-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/notes-to-sefa-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/notes-to-sefa-147110.xlsx index 30e23559ca..33a547257a 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/notes-to-sefa-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/notes-to-sefa-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/secondary-auditors-147110.xlsx b/backend/audit/fixtures/workbooks/should_pass/147110-22/secondary-auditors-147110.xlsx index f0c6dba0f2..f981328964 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/147110-22/secondary-auditors-147110.xlsx and b/backend/audit/fixtures/workbooks/should_pass/147110-22/secondary-auditors-147110.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/147110-22/test-array-147110.json b/backend/audit/fixtures/workbooks/should_pass/147110-22/test-array-147110.json index 507e878684..5e6ba548ef 100644 --- a/backend/audit/fixtures/workbooks/should_pass/147110-22/test-array-147110.json +++ b/backend/audit/fixtures/workbooks/should_pass/147110-22/test-array-147110.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [], "singletons": { "auditee_uei": "ZUFJEU4NMK52" @@ -9,7 +9,7 @@ }, { "endpoint": "additional_ueis", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -234,7 +234,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -1203,7 +1203,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -10737,7 +10737,7 @@ }, { "endpoint": "findings_text", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -11706,7 +11706,7 @@ }, { "endpoint": "findings", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -19985,7 +19985,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [ { "fields": [ @@ -20057,7 +20057,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2022TEST000147110", + "report_id": "2022-06-TSTDAT-0000147110", "rows": [], "singletons": { "auditee_uei": "ZUFJEU4NMK52" diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-eins-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-eins-171944.xlsx index 338923831e..dd30eff613 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-eins-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-eins-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-ueis-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-ueis-171944.xlsx index 7ec6044fcf..97e7589d8e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-ueis-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/additional-ueis-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-171944.xlsx index a51ca76582..bcbb731c85 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-text-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-text-171944.xlsx index 496de9abe6..2443347d12 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-text-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/audit-findings-text-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/corrective-action-plan-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/corrective-action-plan-171944.xlsx index c9329dae32..9747da4455 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/corrective-action-plan-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/corrective-action-plan-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/federal-awards-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/federal-awards-171944.xlsx new file mode 100644 index 0000000000..64c70b4a07 Binary files /dev/null and b/backend/audit/fixtures/workbooks/should_pass/171944-22/federal-awards-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/notes-to-sefa-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/notes-to-sefa-171944.xlsx index a4d10dd1cf..3f40211b2c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/notes-to-sefa-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/notes-to-sefa-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/secondary-auditors-171944.xlsx b/backend/audit/fixtures/workbooks/should_pass/171944-22/secondary-auditors-171944.xlsx index 502d236114..c0d9ef5bb8 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/171944-22/secondary-auditors-171944.xlsx and b/backend/audit/fixtures/workbooks/should_pass/171944-22/secondary-auditors-171944.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/171944-22/test-array-171944.json b/backend/audit/fixtures/workbooks/should_pass/171944-22/test-array-171944.json index 2d77e50cc3..7ab04fcd34 100644 --- a/backend/audit/fixtures/workbooks/should_pass/171944-22/test-array-171944.json +++ b/backend/audit/fixtures/workbooks/should_pass/171944-22/test-array-171944.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -1202,7 +1202,7 @@ }, { "endpoint": "additional_ueis", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -2171,7 +2171,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -2540,7 +2540,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -168186,7 +168186,7 @@ }, { "endpoint": "findings_text", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -168555,7 +168555,7 @@ }, { "endpoint": "findings", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -172770,7 +172770,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ @@ -172932,7 +172932,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2022TEST000171944", + "report_id": "2022-08-TSTDAT-0000171944", "rows": [ { "fields": [ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-eins-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-eins-181744.xlsx index 700a94c0cf..534abb1e93 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-eins-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-eins-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-ueis-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-ueis-181744.xlsx index fcc322254b..cc2efaf75e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-ueis-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/additional-ueis-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-181744.xlsx index b756f86966..87d5dc1026 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-text-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-text-181744.xlsx index 289fa821a5..93aac9efea 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-text-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/audit-findings-text-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/corrective-action-plan-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/corrective-action-plan-181744.xlsx index c946b2ce3c..3ef5c81496 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/corrective-action-plan-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/corrective-action-plan-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/federal-awards-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/federal-awards-181744.xlsx index 847bb4822a..90b5fb35f4 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/federal-awards-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/federal-awards-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/notes-to-sefa-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/notes-to-sefa-181744.xlsx index b21aa0f698..407b35bf86 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/notes-to-sefa-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/notes-to-sefa-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/secondary-auditors-181744.xlsx b/backend/audit/fixtures/workbooks/should_pass/181744-22/secondary-auditors-181744.xlsx index 717e32d0a4..a4bab1533e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/181744-22/secondary-auditors-181744.xlsx and b/backend/audit/fixtures/workbooks/should_pass/181744-22/secondary-auditors-181744.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/181744-22/test-array-181744.json b/backend/audit/fixtures/workbooks/should_pass/181744-22/test-array-181744.json index e76b555a69..436e6d03e3 100644 --- a/backend/audit/fixtures/workbooks/should_pass/181744-22/test-array-181744.json +++ b/backend/audit/fixtures/workbooks/should_pass/181744-22/test-array-181744.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": { "auditee_uei": "MKEJWVSEURF3" @@ -9,7 +9,7 @@ }, { "endpoint": "additional_ueis", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": { "auditee_uei": "MKEJWVSEURF3" @@ -17,7 +17,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": { "auditee_uei": "MKEJWVSEURF3" @@ -25,7 +25,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [ { "fields": [ @@ -5075,7 +5075,7 @@ }, { "endpoint": "findings_text", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": { "auditee_uei": "MKEJWVSEURF3" @@ -5083,13 +5083,13 @@ }, { "endpoint": "findings", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": {} }, { "endpoint": "notes_to_sefa", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [ { "fields": [ @@ -5111,7 +5111,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2022TEST000181744", + "report_id": "2022-09-TSTDAT-0000181744", "rows": [], "singletons": { "auditee_uei": "MKEJWVSEURF3" diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-eins-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-eins-182926.xlsx index 555ed05d4b..c24141418a 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-eins-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-eins-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-ueis-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-ueis-182926.xlsx index 9be1956d50..a8053b8378 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-ueis-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/additional-ueis-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-182926.xlsx index a4c1a07ed4..4e1e50769e 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-text-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-text-182926.xlsx index 39193f27f3..5bdff9508c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-text-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/audit-findings-text-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/corrective-action-plan-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/corrective-action-plan-182926.xlsx index 92d30e9cfb..2128cdae0d 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/corrective-action-plan-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/corrective-action-plan-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/federal-awards-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/federal-awards-182926.xlsx index 2fad1d0640..0b167fa13a 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/federal-awards-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/federal-awards-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/notes-to-sefa-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/notes-to-sefa-182926.xlsx index 0388eec358..f6631f3074 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/notes-to-sefa-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/notes-to-sefa-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/secondary-auditors-182926.xlsx b/backend/audit/fixtures/workbooks/should_pass/182926-22/secondary-auditors-182926.xlsx index fae9eb51b5..d6676be3d1 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/182926-22/secondary-auditors-182926.xlsx and b/backend/audit/fixtures/workbooks/should_pass/182926-22/secondary-auditors-182926.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/182926-22/test-array-182926.json b/backend/audit/fixtures/workbooks/should_pass/182926-22/test-array-182926.json index e16c0c559e..b22a271f2a 100644 --- a/backend/audit/fixtures/workbooks/should_pass/182926-22/test-array-182926.json +++ b/backend/audit/fixtures/workbooks/should_pass/182926-22/test-array-182926.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [], "singletons": { "auditee_uei": "M1LVPE5GLSD9" @@ -9,7 +9,7 @@ }, { "endpoint": "additional_ueis", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [], "singletons": { "auditee_uei": "M1LVPE5GLSD9" @@ -17,7 +17,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [ { "fields": [ @@ -62,7 +62,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [ { "fields": [ @@ -89594,7 +89594,7 @@ }, { "endpoint": "findings_text", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [ { "fields": [ @@ -89639,7 +89639,7 @@ }, { "endpoint": "findings", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [ { "fields": [ @@ -89860,7 +89860,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [ { "fields": [ @@ -89892,7 +89892,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2022TEST000182926", + "report_id": "2022-06-TSTDAT-0000182926", "rows": [], "singletons": { "auditee_uei": "M1LVPE5GLSD9" diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-eins-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-eins-191734.xlsx index 0af3c12761..f890d172f1 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-eins-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-eins-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-ueis-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-ueis-191734.xlsx index cd1f8cd562..0b60726911 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-ueis-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/additional-ueis-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-191734.xlsx index 356d27734d..a8e5ba1bbd 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-text-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-text-191734.xlsx index 6f26d3893b..3946cd0b15 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-text-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/audit-findings-text-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/corrective-action-plan-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/corrective-action-plan-191734.xlsx index 361456a1dc..a8000261a7 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/corrective-action-plan-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/corrective-action-plan-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/federal-awards-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/federal-awards-191734.xlsx index 1469ae30bf..d1915b70be 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/federal-awards-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/federal-awards-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/notes-to-sefa-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/notes-to-sefa-191734.xlsx index abaed2bcfd..6e5eb071b2 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/notes-to-sefa-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/notes-to-sefa-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/secondary-auditors-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-21/secondary-auditors-191734.xlsx index fefcba1070..8f6bde3d1c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-21/secondary-auditors-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-21/secondary-auditors-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-21/test-array-191734.json b/backend/audit/fixtures/workbooks/should_pass/191734-21/test-array-191734.json index 2cb876fcd9..2c58022f6e 100644 --- a/backend/audit/fixtures/workbooks/should_pass/191734-21/test-array-191734.json +++ b/backend/audit/fixtures/workbooks/should_pass/191734-21/test-array-191734.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" @@ -14,7 +14,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -143,7 +143,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -3313,7 +3313,7 @@ }, { "endpoint": "findings_text", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -3442,7 +3442,7 @@ }, { "endpoint": "findings", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -13963,7 +13963,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -13995,7 +13995,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2021TEST000191734", + "report_id": "2021-09-TSTDAT-0000191734", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-eins-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-eins-191734.xlsx index 0042c84b30..bd109286fb 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-eins-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-eins-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-ueis-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-ueis-191734.xlsx index b22207d310..ade49b29c9 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-ueis-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/additional-ueis-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-191734.xlsx index f4992b085e..17206ee7f9 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-text-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-text-191734.xlsx index 3d17bfb0c3..8fee872f99 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-text-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/audit-findings-text-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/corrective-action-plan-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/corrective-action-plan-191734.xlsx index e29d0a2b93..3cd6318c54 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/corrective-action-plan-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/corrective-action-plan-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/federal-awards-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/federal-awards-191734.xlsx index e277118698..a738bf42eb 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/federal-awards-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/federal-awards-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/notes-to-sefa-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/notes-to-sefa-191734.xlsx index 83bff31293..1561688ec0 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/notes-to-sefa-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/notes-to-sefa-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/secondary-auditors-191734.xlsx b/backend/audit/fixtures/workbooks/should_pass/191734-22/secondary-auditors-191734.xlsx index 2f55a1b705..0b692a19f9 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/191734-22/secondary-auditors-191734.xlsx and b/backend/audit/fixtures/workbooks/should_pass/191734-22/secondary-auditors-191734.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/191734-22/test-array-191734.json b/backend/audit/fixtures/workbooks/should_pass/191734-22/test-array-191734.json index e4311e0b29..c252e8856d 100644 --- a/backend/audit/fixtures/workbooks/should_pass/191734-22/test-array-191734.json +++ b/backend/audit/fixtures/workbooks/should_pass/191734-22/test-array-191734.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [], "singletons": { "auditee_uei": "XPJZNYCBRLR6" @@ -9,7 +9,7 @@ }, { "endpoint": "additional_ueis", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [], "singletons": { "auditee_uei": "XPJZNYCBRLR6" @@ -17,7 +17,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -62,7 +62,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -3578,7 +3578,7 @@ }, { "endpoint": "findings_text", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -3623,7 +3623,7 @@ }, { "endpoint": "findings", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [ { "fields": [ @@ -7634,7 +7634,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [], "singletons": { "accounting_policies": "Exemption for Indian Tribes.", @@ -7645,7 +7645,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2022TEST000191734", + "report_id": "2022-09-TSTDAT-0000191734", "rows": [], "singletons": { "auditee_uei": "XPJZNYCBRLR6" diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-eins-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-eins-219107.xlsx index 2648f9469d..3e79645fa9 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-eins-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-eins-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-ueis-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-ueis-219107.xlsx index 56c42965d9..deb086611c 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-ueis-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/additional-ueis-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-219107.xlsx index ec3eb4bcae..1f412fd0e7 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-text-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-text-219107.xlsx index 793187eb59..828f91978d 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-text-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/audit-findings-text-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/corrective-action-plan-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/corrective-action-plan-219107.xlsx index 7b25a78da4..427547d8f0 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/corrective-action-plan-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/corrective-action-plan-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/federal-awards-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/federal-awards-219107.xlsx index 6a2a5ba5b9..99b6d93d31 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/federal-awards-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/federal-awards-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/notes-to-sefa-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/notes-to-sefa-219107.xlsx index 5b2cc3592e..394bbd5d7d 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/notes-to-sefa-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/notes-to-sefa-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/secondary-auditors-219107.xlsx b/backend/audit/fixtures/workbooks/should_pass/219107-21/secondary-auditors-219107.xlsx index 47a53fc36f..12950ce7d2 100644 Binary files a/backend/audit/fixtures/workbooks/should_pass/219107-21/secondary-auditors-219107.xlsx and b/backend/audit/fixtures/workbooks/should_pass/219107-21/secondary-auditors-219107.xlsx differ diff --git a/backend/audit/fixtures/workbooks/should_pass/219107-21/test-array-219107.json b/backend/audit/fixtures/workbooks/should_pass/219107-21/test-array-219107.json index c497d3fb25..25a606b905 100644 --- a/backend/audit/fixtures/workbooks/should_pass/219107-21/test-array-219107.json +++ b/backend/audit/fixtures/workbooks/should_pass/219107-21/test-array-219107.json @@ -1,7 +1,7 @@ [ { "endpoint": "additional_eins", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" @@ -14,7 +14,7 @@ }, { "endpoint": "corrective_action_plans", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [ { "fields": [ @@ -35,7 +35,7 @@ }, { "endpoint": "federal_awards", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [ { "fields": [ @@ -12601,7 +12601,7 @@ }, { "endpoint": "findings_text", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [ { "fields": [ @@ -12622,7 +12622,7 @@ }, { "endpoint": "findings", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [ { "fields": [ @@ -14527,7 +14527,7 @@ }, { "endpoint": "notes_to_sefa", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [], "singletons": { "accounting_policies": "1.The accompanying schedule of expenditures of federal awards (Schedule) includes the federal award activity of RI under programs of the federal government for the year ended June 30, 2021.The information in this Schedule is presented in accordance with the requirements of Title 2 U.S. Code of Federal Regulations Part 200, Uniform Administrative Requirements, Cost Principles and Audit Requirements for Federal Awards (Uniform Guidance).Because the Schedule presents only a selected portion of the operations of RI, it is not intended to and does not present the financial position, changes in net assets or cash flows of RI. Expenditures reported on the Schedule are reported on the accrual basis of accounting.Such expenditures are recognized following the cost principles contained in the Uniform Guidance, wherein certain types of expenditures are not allowable or are limited as to reimbursement.Negative amounts shown on the Schedule represent adjustments or credits made in the normal course of business to amounts reported as expenditures in prior years.", @@ -14538,7 +14538,7 @@ }, { "endpoint": "secondary_auditor", - "report_id": "2021TEST000219107", + "report_id": "2021-06-TSTDAT-0000219107", "rows": [], "singletons": { "auditee_uei": "BADBADBADBAD" diff --git a/backend/audit/intakelib/checks/check_additional_award_identification_present.py b/backend/audit/intakelib/checks/check_additional_award_identification_present.py index 002fe5724c..2b72a4f785 100644 --- a/backend/audit/intakelib/checks/check_additional_award_identification_present.py +++ b/backend/audit/intakelib/checks/check_additional_award_identification_present.py @@ -8,7 +8,7 @@ REGEX_RD_EXTENSION, REGEX_U_EXTENSION, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) @@ -21,7 +21,9 @@ def additional_award_identification(ir): errors = [] patterns = [REGEX_RD_EXTENSION, REGEX_U_EXTENSION] for index, (ext, add) in enumerate(zip(extension, additional)): - if any(re.match(pattern, ext) for pattern in patterns) and not add: + if any(re.match(pattern, ext) for pattern in patterns) and ( + (add is None) or (str(add).strip() == "") + ): errors.append( build_cell_error_tuple( ir, diff --git a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py index ed580ba2d5..d594064edd 100644 --- a/backend/audit/intakelib/checks/check_all_unique_award_numbers.py +++ b/backend/audit/intakelib/checks/check_all_unique_award_numbers.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py index f5711ad78f..1bf908002e 100644 --- a/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py +++ b/backend/audit/intakelib/checks/check_aln_three_digit_extension_pattern.py @@ -4,7 +4,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) @@ -25,7 +25,6 @@ def aln_three_digit_extension(ir): patterns = [REGEX_RD_EXTENSION, REGEX_THREE_DIGIT_EXTENSION, REGEX_U_EXTENSION] for index, ext in enumerate(extension): # Check if ext does not match any of the regex patterns - # Handles None coming in by casting ext to `str` if not any(re.match(pattern, str(ext)) for pattern in patterns): errors.append( build_cell_error_tuple( diff --git a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py index cef39db826..fdbedcb26b 100644 --- a/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py +++ b/backend/audit/intakelib/checks/check_cardinality_of_passthrough_names_and_ids.py @@ -2,7 +2,7 @@ from audit.intakelib.intermediate_representation import ( get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_cluster_name_always_present.py b/backend/audit/intakelib/checks/check_cluster_name_always_present.py deleted file mode 100644 index c8b53d2d14..0000000000 --- a/backend/audit/intakelib/checks/check_cluster_name_always_present.py +++ /dev/null @@ -1,30 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_values_by_name, - get_range_by_name, -) -from .util import get_message, build_cell_error_tuple - -logger = logging.getLogger(__name__) - - -# DESCRIPTION -# The cluster name should always be present. It is either -# a cluster name or `N/A`, but never empty. -# WHY -# People sometimes leave it empty instead of N/A. -def cluster_name_always_present(ir): - cluster_names = get_range_values_by_name(ir, "cluster_name") - errors = [] - for index, cluster_name in enumerate(cluster_names): - if (cluster_name is None) or (str(cluster_name).strip() == ""): - errors.append( - build_cell_error_tuple( - ir, - get_range_by_name(ir, "cluster_name"), - index, - get_message("check_cluster_name_always_present"), - ) - ) - - return errors diff --git a/backend/audit/intakelib/checks/check_cluster_total.py b/backend/audit/intakelib/checks/check_cluster_total.py index 53e3945f19..fadeb51fff 100644 --- a/backend/audit/intakelib/checks/check_cluster_total.py +++ b/backend/audit/intakelib/checks/check_cluster_total.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_direct_award_is_not_blank.py b/backend/audit/intakelib/checks/check_direct_award_is_not_blank.py deleted file mode 100644 index 6ac8666da4..0000000000 --- a/backend/audit/intakelib/checks/check_direct_award_is_not_blank.py +++ /dev/null @@ -1,28 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_values_by_name, - get_range_by_name, -) -from .util import get_message, build_cell_error_tuple - -logger = logging.getLogger(__name__) - - -# DESCRIPTION -# Users sometimes leave this column blank. -# It must always be present. -def direct_award_is_not_blank(ir): - is_direct = get_range_values_by_name(ir, "is_direct") - errors = [] - for index, y_or_n in enumerate(is_direct): - if (y_or_n is None) or (str(y_or_n).strip() == ""): - errors.append( - build_cell_error_tuple( - ir, - get_range_by_name(ir, "is_direct"), - index, - get_message("check_direct_award_is_not_blank"), - ) - ) - - return errors diff --git a/backend/audit/intakelib/checks/check_eins_are_not_empty.py b/backend/audit/intakelib/checks/check_eins_are_not_empty.py index ec362efc91..6c0254aef2 100644 --- a/backend/audit/intakelib/checks/check_eins_are_not_empty.py +++ b/backend/audit/intakelib/checks/check_eins_are_not_empty.py @@ -1,22 +1,13 @@ from django.core.exceptions import ValidationError import logging -from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_missing_value_errors logger = logging.getLogger(__name__) # FIXME: We need comments on all the validations? def eins_are_not_empty(ir): - addl_eins = get_range_by_name(ir, "additional_ein") - errors = [] - for index, ein in enumerate(addl_eins["values"]): - if ein is None: - errors.append( - build_cell_error_tuple( - ir, addl_eins, index, get_message("check_eins_are_not_empty") - ) - ) + errors = get_missing_value_errors(ir, "additional_ein", "check_eins_are_not_empty") if len(errors) > 0: logger.info("Raising a validation error.") raise ValidationError(errors) diff --git a/backend/audit/intakelib/checks/check_federal_award_passed_always_present.py b/backend/audit/intakelib/checks/check_federal_award_passed_always_present.py deleted file mode 100644 index 1811f20bc2..0000000000 --- a/backend/audit/intakelib/checks/check_federal_award_passed_always_present.py +++ /dev/null @@ -1,27 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_values_by_name, - get_range_by_name, -) -from .util import get_message, build_cell_error_tuple - -logger = logging.getLogger(__name__) - - -# DESCRIPTION -# This must always be present. -def federal_award_passed_always_present(ir): - is_passed = get_range_values_by_name(ir, "is_passed") - errors = [] - for index, y_or_n in enumerate(is_passed): - if (y_or_n is None) or (str(y_or_n).strip() == ""): - errors.append( - build_cell_error_tuple( - ir, - get_range_by_name(ir, "is_passed"), - index, - get_message("check_federal_award_passed_always_present"), - ) - ) - - return errors diff --git a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py similarity index 95% rename from backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py rename to backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py index d28e043d53..449bea23fe 100644 --- a/backend/audit/intakelib/checks/check_federal_award_passed_passed_through_optional.py +++ b/backend/audit/intakelib/checks/check_federal_award_passed_through_optional.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_federal_program_total.py b/backend/audit/intakelib/checks/check_federal_program_total.py index d7389f418f..77e52e394f 100644 --- a/backend/audit/intakelib/checks/check_federal_program_total.py +++ b/backend/audit/intakelib/checks/check_federal_program_total.py @@ -3,7 +3,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py new file mode 100644 index 0000000000..c9f631929f --- /dev/null +++ b/backend/audit/intakelib/checks/check_finding_prior_references_pattern.py @@ -0,0 +1,44 @@ +import logging +from django.core.exceptions import ValidationError +import re +from audit.intakelib.intermediate_representation import ( + get_range_by_name, +) +from audit.intakelib.common import ( + get_message, + build_cell_error_tuple, + appears_empty, + is_value_marked_na, +) + +logger = logging.getLogger(__name__) + +# A version of these regexes also exists in Base.libsonnet +PRIOR_REFERENCES_REGEX = r"^20[1-9][0-9]-[0-9]{3}(,\s*20[1-9][0-9]-[0-9]{3})*$" + + +# DESCRIPTION +# Prior references should be a comma separated list of 20##-### values +# TESTED BY +# has_bad_prior_references.xlsx +def prior_references_pattern(ir): + prior_references = get_range_by_name(ir, "prior_references") + errors = [] + for index, prior_reference in enumerate(prior_references["values"]): + if ( + not appears_empty(prior_reference) + and (not is_value_marked_na(prior_reference)) + and (not re.match(PRIOR_REFERENCES_REGEX, str(prior_reference))) + ): + errors.append( + build_cell_error_tuple( + ir, + prior_references, + index, + get_message("check_prior_references_invalid"), + ) + ) + + if len(errors) > 0: + logger.info("Raising a validation error.") + raise ValidationError(errors) diff --git a/backend/audit/intakelib/checks/check_findings_grid_validation.py b/backend/audit/intakelib/checks/check_findings_grid_validation.py index b0daca947c..d22f0eebf6 100644 --- a/backend/audit/intakelib/checks/check_findings_grid_validation.py +++ b/backend/audit/intakelib/checks/check_findings_grid_validation.py @@ -2,7 +2,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple # Modified Opinion # Other Matters @@ -18,6 +18,7 @@ # WHY # It's in the UG. def findings_grid_validation(ir): + # A copy of allowed_combos is in dissemination/workbooklib/findings.py # Values copied directly out of the UG allowed_combos = [ "YNNNN", diff --git a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py index 922b5d7997..e4ba189044 100644 --- a/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py +++ b/backend/audit/intakelib/checks/check_has_all_the_named_ranges.py @@ -3,7 +3,7 @@ from audit.intakelib.intermediate_representation import ( extract_workbook_as_ir, ) -from .util import get_names_of_all_ranges +from audit.intakelib.common import get_names_of_all_ranges from audit.fixtures.excel import FORM_SECTIONS from audit.fixtures.excel import ( ADDITIONAL_UEIS_TEMPLATE, diff --git a/backend/audit/intakelib/checks/check_is_a_workbook.py b/backend/audit/intakelib/checks/check_is_a_workbook.py index b7747315f7..42f46040bb 100644 --- a/backend/audit/intakelib/checks/check_is_a_workbook.py +++ b/backend/audit/intakelib/checks/check_is_a_workbook.py @@ -1,6 +1,9 @@ from django.core.exceptions import ValidationError import logging -from audit.intakelib.intermediate_representation import get_sheet_by_name +from audit.intakelib.intermediate_representation import ( + get_sheet_by_name, + get_range_by_name, +) logger = logging.getLogger(__name__) @@ -12,7 +15,8 @@ # sloppy and still have a coversheet page. def is_a_workbook(ir): coversheet = get_sheet_by_name(ir, "Coversheet") - if not coversheet: + version_range = get_range_by_name(ir, "version") + if not (coversheet and version_range): raise ValidationError( ( "(O_o)", diff --git a/backend/audit/intakelib/checks/check_is_right_workbook.py b/backend/audit/intakelib/checks/check_is_right_workbook.py index 8ef87f12a3..81d7bc5a2e 100644 --- a/backend/audit/intakelib/checks/check_is_right_workbook.py +++ b/backend/audit/intakelib/checks/check_is_right_workbook.py @@ -4,7 +4,7 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_loan_balance.py b/backend/audit/intakelib/checks/check_loan_balance_entries.py similarity index 91% rename from backend/audit/intakelib/checks/check_loan_balance.py rename to backend/audit/intakelib/checks/check_loan_balance_entries.py index ace3bd0b95..7b2be52750 100644 --- a/backend/audit/intakelib/checks/check_loan_balance.py +++ b/backend/audit/intakelib/checks/check_loan_balance_entries.py @@ -3,14 +3,14 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) # DESCRIPTION # This makes sure that the loan guarantee is either a numerical value or N/A or an empty string. -def loan_balance(ir): +def loan_balance_entry_is_valid(ir): loan_balance_at_period_end = get_range_values_by_name( ir, "loan_balance_at_audit_period_end" ) diff --git a/backend/audit/intakelib/checks/check_loan_guarantee.py b/backend/audit/intakelib/checks/check_loan_balance_present.py similarity index 64% rename from backend/audit/intakelib/checks/check_loan_guarantee.py rename to backend/audit/intakelib/checks/check_loan_balance_present.py index bebc4002a0..33e882dcf2 100644 --- a/backend/audit/intakelib/checks/check_loan_guarantee.py +++ b/backend/audit/intakelib/checks/check_loan_balance_present.py @@ -3,22 +3,17 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) -def appears_empty(v): - return (v is None) or (str(v).strip() == "") - - # DESCRIPTION -# This makes sure that the loan guarantee Y/N is present. If not, it throws an error. -# If it is present, and it is not a guarantee, AND there is a balance, that is a problem. -# If it is present, and it is a guarantee, there should be a balance. +# If the loan is not a guarantee, AND there is a balance, that is a problem and the application throws an error. +# If the loan is is a guarantee, there should be a balance. # WHY # A guarantee means we want to know how much was guaranteed. -def loan_guarantee(ir): +def loan_balance_present(ir): is_guaranteed = get_range_values_by_name(ir, "is_guaranteed") loan_balance_at_period_end = get_range_values_by_name( ir, "loan_balance_at_audit_period_end" @@ -28,16 +23,6 @@ def loan_guarantee(ir): for index, (guarantee, balance) in enumerate( zip(is_guaranteed, loan_balance_at_period_end) ): - if appears_empty(guarantee): - errors.append( - build_cell_error_tuple( - ir, - get_range_by_name(ir, "is_guaranteed"), - index, - get_message("check_loan_guarantee_not_empty"), - ) - ) - if (guarantee == "N") and balance: errors.append( build_cell_error_tuple( diff --git a/backend/audit/intakelib/checks/check_look_for_empty_rows.py b/backend/audit/intakelib/checks/check_look_for_empty_rows.py index 437e0e2a9f..a7a805265e 100644 --- a/backend/audit/intakelib/checks/check_look_for_empty_rows.py +++ b/backend/audit/intakelib/checks/check_look_for_empty_rows.py @@ -2,7 +2,7 @@ import logging from audit.intakelib.intermediate_representation import ranges_to_rows, appears_empty -from .util import get_range_start_row +from audit.intakelib.common import get_range_start_row logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_missing_award_numbers.py b/backend/audit/intakelib/checks/check_missing_award_numbers.py deleted file mode 100644 index 7cc93dc222..0000000000 --- a/backend/audit/intakelib/checks/check_missing_award_numbers.py +++ /dev/null @@ -1,23 +0,0 @@ -from django.core.exceptions import ValidationError -import logging -from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple - -logger = logging.getLogger(__name__) - - -# FIXME: We need comments on all the validations? -def missing_award_numbers(ir): - ars = get_range_by_name(ir, "award_reference") - errors = [] - # FIXME: Get rid of all abbreviated variable names. - for index, award_number in enumerate(ars["values"]): - if award_number is None: - errors.append( - build_cell_error_tuple( - ir, ars, index, get_message("check_missing_award_numbers") - ) - ) - if len(errors) > 0: - logger.info("Raising a validation error.") - raise ValidationError(errors) diff --git a/backend/audit/intakelib/checks/check_missing_required_fields.py b/backend/audit/intakelib/checks/check_missing_required_fields.py new file mode 100644 index 0000000000..8f2d37a7cc --- /dev/null +++ b/backend/audit/intakelib/checks/check_missing_required_fields.py @@ -0,0 +1,69 @@ +from django.core.exceptions import ValidationError +import logging +from audit.intakelib.common import get_missing_value_errors +from audit.fixtures.excel import FORM_SECTIONS + +logger = logging.getLogger(__name__) + +map_required_field_ranges_to_workbook = { + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: { + "award_reference": "check_missing_award_numbers", + "federal_agency_prefix": "check_missing_federal_agency_prefix", + "three_digit_extension": "check_missing_aln_three_digit_extension", + "program_name": "check_missing_program_name", + "amount_expended": "check_missing_amount_expended", + "cluster_name": "check_missing_cluster_name", + "federal_program_total": "check_missing_federal_program_total", + "cluster_total": "check_missing_cluster_total", + "is_guaranteed": "check_missing_loan_guaranteed", + "is_direct": "check_direct_award_is_not_blank", + "is_passed": "check_federal_award_passed_always_present", + "is_major": "check_no_major_program_is_blank", + "number_of_audit_findings": "check_num_findings_always_present", + }, + # This will not prevent user for submitting an empty workbook as long as the coversheet is filled out. + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: { + "award_reference": "check_missing_award_numbers", + "reference_number": "check_missing_reference_numbers", + "compliance_requirement": "check_missing_compliance_requirement", + "modified_opinion": "check_missing_modified_opinion", + "other_matters": "check_missing_other_matters", + "material_weakness": "check_missing_material_weakness", + "significant_deficiency": "check_missing_significant_deficiency", + "other_findings": "check_missing_other_findings", + "questioned_costs": "check_missing_questioned_costs", + "repeat_prior_reference": "check_missing_repeat_prior_reference", + "prior_references": "check_missing_prior_references", + "is_valid": "check_missing_is_valid", + }, + # FIXME: MSHD - will add the sections below in follow up PRs. + # FORM_SECTIONS.ADDITIONAL_EINS:{ + # "additional_ein": "check_eins_are_not_empty", + # }, + # FORM_SECTIONS.ADDITIONAL_UEIS:{ + # "additional_uei": "check_ueis_are_not_empty", + # }, + # FORM_SECTIONS.NOTES_TO_SEFA:{ + # "is_minimis_rate_used": "check_minimis_rate_used_is_not_blank", + # "rate_explained": "check_rate_explained_is_not_blank", + # "accounting_policies": "check_accounting_policies_is_not_blank", + # } + # FORM_SECTIONS.SECONDARY_AUDITORS: + # FORM_SECTIONS.FINDINGS_TEXT: + # FORM_SECTIONS.CORRECTIVE_ACTION_PLAN: +} + + +# check if any required field is missing +def has_all_required_fields(section_name): + def _missing_required_fields(ir): + required_field_ranges = map_required_field_ranges_to_workbook[section_name] + errors = [] + for range_name, error_message_name in required_field_ranges.items(): + errors.extend(get_missing_value_errors(ir, range_name, error_message_name)) + + if len(errors) > 0: + logger.info("Raising a validation error.") + raise ValidationError(errors) + + return _missing_required_fields diff --git a/backend/audit/intakelib/checks/check_no_major_program_no_type.py b/backend/audit/intakelib/checks/check_no_major_program_no_type.py index db9365e4ae..a11dd02f5a 100644 --- a/backend/audit/intakelib/checks/check_no_major_program_no_type.py +++ b/backend/audit/intakelib/checks/check_no_major_program_no_type.py @@ -1,14 +1,10 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) -def appears_empty(v): - return (v is None) or (str(v).strip() == "") - - def no_major_program_no_type(ir): is_major = get_range_by_name(ir, "is_major") audit_report_type = get_range_by_name(ir, "audit_report_type") @@ -18,16 +14,7 @@ def no_major_program_no_type(ir): for ndx, (is_m, rep_type) in enumerate( zip(is_major["values"], audit_report_type["values"]) ): - if appears_empty(is_m): - errors.append( - build_cell_error_tuple( - ir, - audit_report_type, - ndx, - get_message("check_no_major_program_is_blank"), - ) - ) - elif (is_m == "N") and rep_type: + if (is_m == "N") and rep_type: errors.append( build_cell_error_tuple( ir, diff --git a/backend/audit/intakelib/checks/check_no_repeat_findings.py b/backend/audit/intakelib/checks/check_no_repeat_findings.py index 1652c98a1c..7f45a958f8 100644 --- a/backend/audit/intakelib/checks/check_no_repeat_findings.py +++ b/backend/audit/intakelib/checks/check_no_repeat_findings.py @@ -1,6 +1,10 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import ( + get_message, + build_cell_error_tuple, + is_value_marked_na, +) logger = logging.getLogger(__name__) @@ -13,7 +17,7 @@ def no_repeat_findings(ir): for ndx, (is_rep, prior) in enumerate( zip(repeat_prior_reference["values"], prior_references["values"]) ): - if (is_rep == "N") and (prior != "N/A"): + if (is_rep == "N") and (not is_value_marked_na(prior)): errors.append( build_cell_error_tuple( ir, @@ -22,7 +26,7 @@ def no_repeat_findings(ir): get_message("check_no_repeat_findings_when_n"), ) ) - elif (is_rep == "Y") and ((not prior) or (prior == "N/A")): + elif (is_rep == "Y") and ((not prior) or (is_value_marked_na(prior))): errors.append( build_cell_error_tuple( ir, diff --git a/backend/audit/intakelib/checks/check_num_findings_always_present.py b/backend/audit/intakelib/checks/check_num_findings_always_present.py deleted file mode 100644 index ef57b44274..0000000000 --- a/backend/audit/intakelib/checks/check_num_findings_always_present.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple - -logger = logging.getLogger(__name__) - - -def num_findings_always_present(ir): - noaf = get_range_by_name(ir, "number_of_audit_findings") - errors = [] - for ndx, v in enumerate(noaf["values"]): - if (v is None) or (str(v).strip() == ""): - errors.append( - build_cell_error_tuple( - ir, noaf, ndx, get_message("check_num_findings_always_present") - ) - ) - - return errors diff --git a/backend/audit/intakelib/checks/check_other_cluster_names.py b/backend/audit/intakelib/checks/check_other_cluster_names.py index 85a0ba6bd1..57de4dd0e6 100644 --- a/backend/audit/intakelib/checks/check_other_cluster_names.py +++ b/backend/audit/intakelib/checks/check_other_cluster_names.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py index de11e8ab87..c63a9c68dd 100644 --- a/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py +++ b/backend/audit/intakelib/checks/check_passthrough_name_when_no_direct.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_sequential_award_numbers.py b/backend/audit/intakelib/checks/check_sequential_award_numbers.py index 06bae0bbd1..850f3f26c8 100644 --- a/backend/audit/intakelib/checks/check_sequential_award_numbers.py +++ b/backend/audit/intakelib/checks/check_sequential_award_numbers.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple import re logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_state_cluster_names.py b/backend/audit/intakelib/checks/check_state_cluster_names.py index f3f692e5f2..8e984eebbc 100644 --- a/backend/audit/intakelib/checks/check_state_cluster_names.py +++ b/backend/audit/intakelib/checks/check_state_cluster_names.py @@ -1,6 +1,6 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_total_amount_expended.py b/backend/audit/intakelib/checks/check_total_amount_expended.py index e9a041bed9..2827adc671 100644 --- a/backend/audit/intakelib/checks/check_total_amount_expended.py +++ b/backend/audit/intakelib/checks/check_total_amount_expended.py @@ -3,14 +3,14 @@ get_range_values_by_name, get_range_by_name, ) -from .util import get_message, build_cell_error_tuple +from audit.intakelib.common import get_message, build_cell_error_tuple logger = logging.getLogger(__name__) # DESCRIPTION # The sum of the amount_expended should equal the total_amount_expended -# B5=SUM(Form!F$2:F$5000) +# B5=SUM(Form!F$2:F$MAX_ROWS) def total_amount_expended_is_correct(ir): total_amount_expended_value = get_range_values_by_name(ir, "total_amount_expended") amount_expended_values = get_range_values_by_name(ir, "amount_expended") diff --git a/backend/audit/intakelib/checks/check_uei_exists.py b/backend/audit/intakelib/checks/check_uei_exists.py index 0648d2679f..bc88ec839f 100644 --- a/backend/audit/intakelib/checks/check_uei_exists.py +++ b/backend/audit/intakelib/checks/check_uei_exists.py @@ -1,6 +1,10 @@ import logging from audit.intakelib.intermediate_representation import get_range_by_name -from .util import list_contains_non_null_values, get_message, build_range_error_tuple +from audit.intakelib.common import ( + list_contains_non_null_values, + get_message, + build_range_error_tuple, +) logger = logging.getLogger(__name__) diff --git a/backend/audit/intakelib/checks/check_version_number.py b/backend/audit/intakelib/checks/check_version_number.py new file mode 100644 index 0000000000..f1ab04e63f --- /dev/null +++ b/backend/audit/intakelib/checks/check_version_number.py @@ -0,0 +1,32 @@ +from django.core.exceptions import ValidationError +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, +) +from audit.intakelib.common import get_message, build_cell_error_tuple + +logger = logging.getLogger(__name__) + +AUTHORIZED_VERSIONS = {"1.0.0", "1.0.1", "1.0.2", "1.0.3"} + + +# DESCRIPTION +# This checks if the uploaded workbook version is valid. +def validate_workbook_version(ir): + version_range = get_range_by_name(ir, "version") + errors = [] + for index, version in enumerate(version_range["values"]): + # Check if version is not in the set of valid versions + if version not in AUTHORIZED_VERSIONS: + errors.append( + build_cell_error_tuple( + ir, + version_range, + index, + get_message("check_workbook_version").format(version), + ) + ) + + if errors: + logger.info("Raising a validation error.") + raise ValidationError(errors) diff --git a/backend/audit/intakelib/checks/check_y_or_n__fields.py b/backend/audit/intakelib/checks/check_y_or_n__fields.py new file mode 100644 index 0000000000..0af2674761 --- /dev/null +++ b/backend/audit/intakelib/checks/check_y_or_n__fields.py @@ -0,0 +1,42 @@ +import logging +from audit.intakelib.common import invalid_y_or_n_entry +from audit.fixtures.excel import FORM_SECTIONS + +logger = logging.getLogger(__name__) + +map_yorn_field_ranges_to_workbook = { + FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: [ + "is_guaranteed", + "is_direct", + "is_passed", + "is_major", + ], + FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: [ + "modified_opinion", + "other_matters", + "material_weakness", + "significant_deficiency", + "other_findings", + "questioned_costs", + "repeat_prior_reference", + "is_valid", + ], + # FIXME: MSHD - will add the sections below in follow up PRs. + # FORM_SECTIONS.FINDINGS_TEXT:["contains_chart_or_table"], + # FORM_SECTIONS.CORRECTIVE_ACTION_PLAN:["contains_chart_or_table"] +} + + +# check if any Y/N field is invalid +def has_invalid_yorn_field(section_name): + def _invalid_yorn(ir): + yorn_field_ranges = map_yorn_field_ranges_to_workbook[section_name] + errors = [] + for range_name in yorn_field_ranges: + errors.extend( + invalid_y_or_n_entry(ir, range_name, "check_invalid_y_or_n_entry") + ) + + return errors + + return _invalid_yorn diff --git a/backend/audit/intakelib/checks/runners.py b/backend/audit/intakelib/checks/runners.py index 40a71e3b5e..dde0f917f0 100644 --- a/backend/audit/intakelib/checks/runners.py +++ b/backend/audit/intakelib/checks/runners.py @@ -10,25 +10,19 @@ from .check_start_and_end_rows_of_all_columns_are_same import ( start_and_end_rows_of_all_columns_are_same, ) +from .check_version_number import validate_workbook_version ############ # Federal awards checks from .check_is_right_workbook import is_right_workbook from .check_state_cluster_names import state_cluster_names from .check_other_cluster_names import other_cluster_names -from .check_direct_award_is_not_blank import direct_award_is_not_blank from .check_passthrough_name_when_no_direct import passthrough_name_when_no_direct -from .check_loan_guarantee import loan_guarantee -from .check_loan_balance import loan_balance +from .check_loan_balance_present import loan_balance_present +from .check_loan_balance_entries import loan_balance_entry_is_valid from .check_no_major_program_no_type import no_major_program_no_type -from .check_missing_award_numbers import missing_award_numbers from .check_all_unique_award_numbers import all_unique_award_numbers from .check_sequential_award_numbers import sequential_award_numbers -from .check_num_findings_always_present import num_findings_always_present -from .check_cluster_name_always_present import cluster_name_always_present -from .check_federal_award_passed_always_present import ( - federal_award_passed_always_present, -) from .check_aln_three_digit_extension_pattern import aln_three_digit_extension from .check_additional_award_identification_present import ( additional_award_identification, @@ -36,25 +30,28 @@ from .check_federal_program_total import federal_program_total_is_correct from .check_cluster_total import cluster_total_is_correct from .check_total_amount_expended import total_amount_expended_is_correct -from .check_federal_award_passed_passed_through_optional import ( +from .check_federal_award_passed_through_optional import ( federal_award_amount_passed_through_optional, ) from .check_cardinality_of_passthrough_names_and_ids import ( cardinality_of_passthrough_names_and_ids, ) from .check_has_all_the_named_ranges import has_all_the_named_ranges - +from .check_missing_required_fields import has_all_required_fields +from .check_y_or_n__fields import has_invalid_yorn_field from .check_show_ir import show_ir ############ # Audit findings checks from .check_no_repeat_findings import no_repeat_findings from .check_findings_grid_validation import findings_grid_validation +from .check_finding_prior_references_pattern import prior_references_pattern logger = logging.getLogger(__name__) general_checks = [ is_a_workbook, + validate_workbook_version, uei_exists, look_for_empty_rows, start_and_end_rows_of_all_columns_are_same, @@ -63,17 +60,14 @@ federal_awards_checks = general_checks + [ is_right_workbook(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED), has_all_the_named_ranges(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED), - missing_award_numbers, - num_findings_always_present, - cluster_name_always_present, - federal_award_passed_always_present, + has_all_required_fields(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED), + has_invalid_yorn_field(FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED), federal_award_amount_passed_through_optional, state_cluster_names, other_cluster_names, - direct_award_is_not_blank, passthrough_name_when_no_direct, - loan_balance, - loan_guarantee, + loan_balance_present, + loan_balance_entry_is_valid, no_major_program_no_type, all_unique_award_numbers, sequential_award_numbers, @@ -93,6 +87,9 @@ audit_findings_checks = general_checks + [ is_right_workbook(FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE), has_all_the_named_ranges(FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE), + has_all_required_fields(FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE), + has_invalid_yorn_field(FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE), + prior_references_pattern, no_repeat_findings, findings_grid_validation, ] diff --git a/backend/audit/intakelib/checks/util.py b/backend/audit/intakelib/checks/util.py deleted file mode 100644 index 0457722cc2..0000000000 --- a/backend/audit/intakelib/checks/util.py +++ /dev/null @@ -1,62 +0,0 @@ -from .error_messages import messages -import logging - -logger = logging.getLogger(__name__) - - -def list_contains_non_null_values(ls): - filtered = filter(lambda v: v is not None, ls) - return len(list(filtered)) > 0 - - -def get_message(key): - if key in messages: - return messages[key] - else: - return f"No error message found for {key}. Please contact the helpdesk." - - -def get_range_start_col(range): - return range["start_cell"]["column"] - - -def get_range_start_row(range): - return range["start_cell"]["row"] - - -def get_sheet_name_from_range_name(ir, range_name): - for sheet in ir: - sheet_name = sheet["name"] - for range in sheet["ranges"]: - if range["name"] == range_name: - return sheet_name - return "Uknown sheet" - - -# [(col1, row1, field1, link1, help-text1), (col2, row2, ...), ...] -def build_range_error_tuple(ir, range, range_name, text): - return ( - get_range_start_col(range), - get_range_start_row(range), - get_sheet_name_from_range_name(ir, range_name), - {"text": text, "link": "Intake checks: no link defined"}, - ) - - -def build_cell_error_tuple(ir, range, ndx, message): - return ( - get_range_start_col(range), - int(get_range_start_row(range)) + ndx, - get_sheet_name_from_range_name(ir, range["name"]), - {"text": message, "link": "Intake checks: no link defined"}, - ) - - -def get_names_of_all_ranges(data): - names = [] - for entry in data: - if "ranges" in entry: - for range_item in entry["ranges"]: - if "name" in range_item: - names.append(range_item["name"]) - return names diff --git a/backend/audit/intakelib/common/__init__.py b/backend/audit/intakelib/common/__init__.py new file mode 100644 index 0000000000..665ee1f767 --- /dev/null +++ b/backend/audit/intakelib/common/__init__.py @@ -0,0 +1,14 @@ +# flake8: noqa: F401 +from .util import ( + get_message, + build_cell_error_tuple, + appears_empty, + is_value_marked_na, + get_names_of_all_ranges, + get_range_start_row, + get_missing_value_errors, + invalid_y_or_n_entry, + safe_int_conversion, + list_contains_non_null_values, + build_range_error_tuple, +) diff --git a/backend/audit/intakelib/checks/error_messages.py b/backend/audit/intakelib/common/error_messages.py similarity index 64% rename from backend/audit/intakelib/checks/error_messages.py rename to backend/audit/intakelib/common/error_messages.py index 4f93088ad5..63fff66139 100644 --- a/backend/audit/intakelib/checks/error_messages.py +++ b/backend/audit/intakelib/common/error_messages.py @@ -12,22 +12,41 @@ "check_no_major_program_no_type_when_n": "When major program is N, leave audit report type blank", "check_no_major_program_no_type_when_y": "When major program is Y, audit report type must be chosen", "check_no_repeat_findings_when_n": "When repeat findings from prior years is N, prior references must be N/A", - "check_no_repeat_findings_when_y": "When repeat findings from prior years is is Y, include prior year references", + "check_no_repeat_findings_when_y": "When repeat findings from prior years is Y, include prior year references", "check_missing_award_numbers": "Missing an award number", + "check_missing_reference_numbers": "Missing a reference number", + "check_missing_compliance_requirement": "Missing the type of compliance requirement", + "check_missing_modified_opinion": "Missing modified opinion input", + "check_missing_other_matters": "Missing other matters input", + "check_missing_material_weakness": "Missing material weakness input", + "check_missing_significant_deficiency": "Missing significant deficiency input", + "check_missing_other_findings": "Missing other findings input", + "check_missing_questioned_costs": "Missing questioned costs input", + "check_missing_repeat_prior_reference": "Missing repeat findings from prior year", + "check_missing_prior_references": "Missing prior year audit finding reference numbers", + "check_missing_is_valid": "Missing findings combination validity", + "check_missing_federal_agency_prefix": "Missing federal agency prefix", + "check_missing_program_name": "Missing program name", + "check_missing_amount_expended": "Missing amount expended", + "check_missing_federal_program_total": "Missing federal program total", + "check_missing_cluster_total": "Missing cluster total", "check_all_unique_award_numbers": "Repeated award number", "check_sequential_award_numbers_regex": "Award references should be of the form AWARD-####", "check_sequential_award_numbers_off": "Award reference is {}, but should be {}", "check_num_findings_always_present": "Number of audit findings must zero or greater", - "check_cluster_name_always_present": "Cluster name cannot be blank; select a name or N/A", + "check_missing_cluster_name": "Cluster name cannot be blank; select a name or N/A", "check_passthrough_name_when_yes_direct": "When direct award is Y, no passthrough name required", "check_no_major_program_is_blank": "Major program must be Y or N; cannot be left empty", - "check_loan_guarantee_not_empty": "Loan guarantee must be Y or N; cannot be left empty", + "check_missing_loan_guaranteed": "Loan guarantee must be Y or N; cannot be left empty", "check_federal_award_passed_always_present": "Federal award passed must be Y or N; cannot be left empty", "check_passthrough_name_when_no_direct_n_and_empty_number": "When the award is direct, passthrough number must be empty", "check_findings_grid_validation": "The combination of findings {} is not a valid combination under Uniform Guidance", "check_eins_are_not_empty": "EIN cannot be empty", - "check_aln_three_digit_extension_missing": "Missing ALN (CFDA) three digit extension", + "check_ueis_are_not_empty": "UEI cannot be empty", + "check_invalid_y_or_n_entry": "{} is not accepted; please enter either Y or N", + "check_missing_aln_three_digit_extension": "Missing ALN (CFDA) three digit extension", "check_aln_three_digit_extension_invalid": "The three digit extension should follow one of these formats: ###, RD#, or U##, where # represents a number", + "check_prior_references_invalid": "Prior references must be N/A or a comma-separated list of values in the format 20##-###, for example, 2019-001, 2019-002", "check_additional_award_identification_present": "Missing additional award identification", "check_federal_program_total": "Federal program total is {}, but should be {}", "check_cluster_total": "This cluster total is {}, but should be {}", @@ -36,4 +55,6 @@ "check_federal_award_amount_passed_through_not_allowed": "When Federal Award Passed Through is N, Amount Passed Through must be empty", "check_loan_balance": "The loan balance is currently set to {}. It should either be a positive number, N/A, or left empty", "check_cardinality_of_passthrough_names_and_ids": "You used a | (bar character) to indicate multiple passthrough names and IDs; you must provide equal numbers of names and IDs. You provided {} name{} and {} ID{}", + "check_workbook_version": "Single audit workbook template version {} is not supported. Please download the latest workbook and transfer your data to it", + "check_integer_values": "{} is not a valid integer", } diff --git a/backend/audit/intakelib/common/util.py b/backend/audit/intakelib/common/util.py new file mode 100644 index 0000000000..42ebfba623 --- /dev/null +++ b/backend/audit/intakelib/common/util.py @@ -0,0 +1,152 @@ +from .error_messages import messages +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, + replace_range_by_name, +) +from django.core.exceptions import ValidationError + +logger = logging.getLogger(__name__) + + +def list_contains_non_null_values(ls): + filtered = filter(lambda v: v is not None, ls) + return len(list(filtered)) > 0 + + +def get_message(key): + if key in messages: + return messages[key] + else: + return f"No error message found for {key}. Please contact the helpdesk." + + +def get_range_start_col(range): + return range["start_cell"]["column"] + + +def get_range_start_row(range): + return range["start_cell"]["row"] + + +def get_sheet_name_from_range_name(ir, range_name): + for sheet in ir: + sheet_name = sheet["name"] + for range in sheet["ranges"]: + if range["name"] == range_name: + return sheet_name + return "Uknown sheet" + + +# [(col1, row1, field1, link1, help-text1), (col2, row2, ...), ...] +def build_range_error_tuple(ir, range, range_name, text): + return ( + get_range_start_col(range), + get_range_start_row(range), + get_sheet_name_from_range_name(ir, range_name), + {"text": text, "link": "Intake checks: no link defined"}, + ) + + +def build_cell_error_tuple(ir, range, ndx, message): + return ( + get_range_start_col(range), + int(get_range_start_row(range)) + ndx, + get_sheet_name_from_range_name(ir, range["name"]), + {"text": message, "link": "Intake checks: no link defined"}, + ) + + +def appears_empty(v): + return (v is None) or (str(v).strip() == "") + + +def is_value_marked_na(v): + value = str(v).strip() + return value == "N/A" + + +def is_y_or_n(v): + value = str(v).strip() + return value == "Y" or value == "N" + + +def get_missing_value_errors(ir, range_name, message_key): + range_data = get_range_by_name(ir, range_name) + errors = [] + if range_data: + for index, value in enumerate(range_data["values"]): + if appears_empty(value): + errors.append( + build_cell_error_tuple( + ir, + range_data, + index, + get_message(message_key), + ) + ) + + return errors + + +def invalid_y_or_n_entry(ir, range_name, message_key): + range_data = get_range_by_name(ir, range_name) + errors = [] + if range_data: + for index, value in enumerate(range_data["values"]): + if not is_y_or_n(value): + errors.append( + build_cell_error_tuple( + ir, + range_data, + index, + get_message(message_key).format(value), + ) + ) + + return errors + + +def get_names_of_all_ranges(data): + names = [] + for entry in data: + if "ranges" in entry: + for range_item in entry["ranges"]: + if "name" in range_item: + names.append(range_item["name"]) + return names + + +def safe_int_conversion(ir, range_name, other_values_allowed=None): + range_data = get_range_by_name(ir, range_name) + errors = [] + new_values = [] + if range_data: + for index, value in enumerate(range_data["values"]): + try: + float_value = float(value) + if float_value.is_integer(): + new_values.append(int(float_value)) + else: + raise ValueError + except (ValueError, TypeError): + # If the value is None, we keep it. This is because some int fields are optional. + # For non optional fields, there is a check for missing required fields that will raise an error. + if (value is None) or ( + other_values_allowed and value in other_values_allowed + ): + new_values.append(value) + else: + errors.append( + build_cell_error_tuple( + ir, + range_data, + index, + get_message("check_integer_values").format(value), + ) + ) + if len(errors) > 0: + logger.info("Raising a validation error.") + raise ValidationError(errors) + new_ir = replace_range_by_name(ir, range_name, new_values) + return new_ir diff --git a/backend/audit/intakelib/intermediate_representation.py b/backend/audit/intakelib/intermediate_representation.py index 70f2cd7a22..dad05a8432 100644 --- a/backend/audit/intakelib/intermediate_representation.py +++ b/backend/audit/intakelib/intermediate_representation.py @@ -281,7 +281,7 @@ def extract_workbook_as_ir(file): sheet["ranges"] = ranges sheets.append(sheet) - # Remove all the Nones at the bottom of the sheets, since we have 5000 rows of formulas. + # Remove all the Nones at the bottom of the sheets, since we have 10000 rows of formulas. for sheet in sheets: remove_null_rows(sheet) diff --git a/backend/audit/intakelib/mapping_additional_eins.py b/backend/audit/intakelib/mapping_additional_eins.py index 2f306ab19e..31180723e4 100644 --- a/backend/audit/intakelib/mapping_additional_eins.py +++ b/backend/audit/intakelib/mapping_additional_eins.py @@ -46,8 +46,8 @@ def extract_additional_eins(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_EINS) - run_all_additional_eins_checks(ir) xform_ir = run_all_additional_eins_transforms(ir) + run_all_additional_eins_checks(xform_ir) result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_additional_ueis.py b/backend/audit/intakelib/mapping_additional_ueis.py index ec3c65b88d..d58696a862 100644 --- a/backend/audit/intakelib/mapping_additional_ueis.py +++ b/backend/audit/intakelib/mapping_additional_ueis.py @@ -22,7 +22,7 @@ ) from .mapping_meta import meta_mapping - +from .transforms import run_all_additional_ueis_transforms from .checks import run_all_general_checks, run_all_additional_ueis_checks logger = logging.getLogger(__name__) @@ -43,8 +43,9 @@ def extract_additional_ueis(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.ADDITIONAL_UEIS) - run_all_additional_ueis_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_additional_ueis_transforms(ir) + run_all_additional_ueis_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_audit_findings.py b/backend/audit/intakelib/mapping_audit_findings.py index 1edc88c491..6b29bd02b8 100644 --- a/backend/audit/intakelib/mapping_audit_findings.py +++ b/backend/audit/intakelib/mapping_audit_findings.py @@ -23,6 +23,7 @@ from .mapping_meta import meta_mapping from .checks import run_all_general_checks, run_all_audit_finding_checks +from .transforms import run_all_audit_findings_transforms logger = logging.getLogger(__name__) @@ -42,8 +43,9 @@ def extract_audit_findings(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE) - run_all_audit_finding_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_audit_findings_transforms(ir) + run_all_audit_finding_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_audit_findings_text.py b/backend/audit/intakelib/mapping_audit_findings_text.py index dcc0887fff..a7817e8e3b 100644 --- a/backend/audit/intakelib/mapping_audit_findings_text.py +++ b/backend/audit/intakelib/mapping_audit_findings_text.py @@ -21,7 +21,7 @@ ) from .mapping_meta import meta_mapping - +from .transforms import run_all_audit_findings_text_transforms from .checks import run_all_general_checks, run_all_audit_findings_text_checks logger = logging.getLogger(__name__) @@ -42,8 +42,9 @@ def extract_audit_findings_text(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.FINDINGS_TEXT) - run_all_audit_findings_text_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_audit_findings_text_transforms(ir) + run_all_audit_findings_text_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_corrective_action_plan.py b/backend/audit/intakelib/mapping_corrective_action_plan.py index 1068f784cb..82ae5b05ae 100644 --- a/backend/audit/intakelib/mapping_corrective_action_plan.py +++ b/backend/audit/intakelib/mapping_corrective_action_plan.py @@ -22,6 +22,7 @@ from .mapping_meta import meta_mapping from .checks import run_all_general_checks, run_all_corrective_action_plan_checks +from .transforms import run_all_corrective_action_plan_transforms logger = logging.getLogger(__name__) @@ -41,8 +42,9 @@ def extract_corrective_action_plan(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.CORRECTIVE_ACTION_PLAN) - run_all_corrective_action_plan_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_corrective_action_plan_transforms(ir) + run_all_corrective_action_plan_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/mapping_secondary_auditors.py b/backend/audit/intakelib/mapping_secondary_auditors.py index 02a3c31d73..acd99a94e3 100644 --- a/backend/audit/intakelib/mapping_secondary_auditors.py +++ b/backend/audit/intakelib/mapping_secondary_auditors.py @@ -18,7 +18,7 @@ from .mapping_meta import meta_mapping from .checks import run_all_general_checks, run_all_secondary_auditors_checks - +from .transforms import run_all_secondary_auditors_transforms from .intermediate_representation import ( extract_workbook_as_ir, _extract_generic_data, @@ -42,8 +42,9 @@ def extract_secondary_auditors(file): ir = extract_workbook_as_ir(file) run_all_general_checks(ir, FORM_SECTIONS.SECONDARY_AUDITORS) - run_all_secondary_auditors_checks(ir) - result = _extract_generic_data(ir, params) + xform_ir = run_all_secondary_auditors_transforms(ir) + run_all_secondary_auditors_checks(xform_ir) + result = _extract_generic_data(xform_ir, params) return result diff --git a/backend/audit/intakelib/transforms/__init__.py b/backend/audit/intakelib/transforms/__init__.py index 836a75a9a6..b59f5eb09f 100644 --- a/backend/audit/intakelib/transforms/__init__.py +++ b/backend/audit/intakelib/transforms/__init__.py @@ -3,4 +3,9 @@ run_all_notes_to_sefa_transforms, run_all_additional_eins_transforms, run_all_federal_awards_transforms, + run_all_additional_ueis_transforms, + run_all_audit_findings_text_transforms, + run_all_audit_findings_transforms, + run_all_corrective_action_plan_transforms, + run_all_secondary_auditors_transforms, ) diff --git a/backend/audit/intakelib/transforms/runners.py b/backend/audit/intakelib/transforms/runners.py index 4a3a0c7b9f..32441ffb25 100644 --- a/backend/audit/intakelib/transforms/runners.py +++ b/backend/audit/intakelib/transforms/runners.py @@ -1,11 +1,30 @@ import logging from copy import deepcopy -from .xform_no_op import no_op - +from .xform_all_amount_expended_need_to_be_integers import ( + convert_amount_expended_to_integers, +) +from .xform_all_cluster_total_need_to_be_integers import ( + convert_cluster_total_to_integers, +) +from .xform_all_federal_program_total_need_to_be_integers import ( + convert_federal_program_total_to_integers, +) +from .xform_total_amount_expended_need_to_be_integers import ( + convert_total_amount_expended_to_integers, +) +from .xform_subrecipient_amount_need_to_be_integers import ( + convert_subrecipient_amount_to_integers, +) from .xform_insert_sequence_nums_into_notes_to_sefa import ( insert_sequence_nums_into_notes_to_sefa, ) +from .xform_number_of_findings_need_to_be_integers import ( + convert_number_of_findings_to_integers, +) +from .xform_loan_balance_need_to_be_integers import ( + convert_loan_balance_to_integers_or_na, +) # from .xform_filter_seq_numbers_where_there_are_no_values import filter_seq_numbers_where_there_are_no_values # from .xform_make_sure_notes_to_sefa_are_just_strings import make_sure_notes_to_sefa_are_just_strings @@ -13,16 +32,15 @@ trim_null_from_content_fields_in_notes_to_sefa, ) -from .xform_eins_need_to_be_strings import eins_need_to_be_strings +from .xform_all_fields_to_stripped_string import convert_to_stripped_string + from .xform_rename_additional_notes_sheet import ( rename_additional_notes_sheet_to_form_sheet, ) -from .xform_all_alns_need_to_be_strings import all_alns_need_to_be_strings -from .xform_all_passthrough_id_need_to_be_strings import ( - all_passthrough_id_need_to_be_strings, -) from .xform_add_transform_for_cfda_key import generate_cfda_keys +from .xform_uniform_cluster_names import regenerate_uniform_cluster_names +from .xform_reformat_prior_references import reformat_prior_references logger = logging.getLogger(__name__) @@ -39,14 +57,36 @@ def run_all_notes_to_sefa_transforms(ir): def run_all_additional_eins_transforms(ir): - return run_all_transforms(ir, additional_eins_transforms) + return run_all_transforms(ir, general_transforms) def run_all_federal_awards_transforms(ir): return run_all_transforms(ir, federal_awards_transforms) -general_transforms = [no_op] +def run_all_additional_ueis_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +def run_all_audit_findings_text_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +def run_all_audit_findings_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +def run_all_corrective_action_plan_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +def run_all_secondary_auditors_transforms(ir): + return run_all_transforms(ir, general_transforms) + + +general_transforms = [ + convert_to_stripped_string, +] notes_to_sefa_transforms = general_transforms + [ trim_null_from_content_fields_in_notes_to_sefa, @@ -54,13 +94,19 @@ def run_all_federal_awards_transforms(ir): insert_sequence_nums_into_notes_to_sefa, ] -additional_eins_transforms = general_transforms + [ - eins_need_to_be_strings, -] - federal_awards_transforms = general_transforms + [ - all_alns_need_to_be_strings, - all_passthrough_id_need_to_be_strings, - # FIXME: Make sure generate_cfda_keys is placed after convert_to_stripped_string when PR #2640 is merged + convert_amount_expended_to_integers, + convert_cluster_total_to_integers, + convert_federal_program_total_to_integers, + convert_total_amount_expended_to_integers, + convert_subrecipient_amount_to_integers, + convert_total_amount_expended_to_integers, + convert_number_of_findings_to_integers, + convert_loan_balance_to_integers_or_na, + regenerate_uniform_cluster_names, generate_cfda_keys, ] + +audit_findings_transforms = general_transforms + [ + reformat_prior_references, +] diff --git a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py deleted file mode 100644 index a60b907e60..0000000000 --- a/backend/audit/intakelib/transforms/xform_all_alns_need_to_be_strings.py +++ /dev/null @@ -1,19 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def all_alns_need_to_be_strings(ir): - agencies = get_range_by_name(ir, "federal_agency_prefix") - new_values = list(map(lambda v: str(v), agencies["values"])) - new_ir = replace_range_by_name(ir, "federal_agency_prefix", new_values) - - extensions = get_range_by_name(ir, "three_digit_extension") - new_values = list(map(lambda v: str(v), extensions["values"])) - new_ir = replace_range_by_name(ir, "three_digit_extension", new_values) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py new file mode 100644 index 0000000000..e7580d0772 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_amount_expended_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all amount expended to integers +def convert_amount_expended_to_integers(ir): + xform_ir = safe_int_conversion(ir, "amount_expended") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py new file mode 100644 index 0000000000..f598dc01e7 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_cluster_total_need_to_be_integers.py @@ -0,0 +1,12 @@ +import logging + +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all cluster totals to integers +def convert_cluster_total_to_integers(ir): + xform_ir = safe_int_conversion(ir, "cluster_total") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py new file mode 100644 index 0000000000..b15831fb1d --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_federal_program_total_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all federal program totals to integers +def convert_federal_program_total_to_integers(ir): + xform_ir = safe_int_conversion(ir, "federal_program_total") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py new file mode 100644 index 0000000000..0d48b503f8 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_all_fields_to_stripped_string.py @@ -0,0 +1,23 @@ +import logging +from copy import deepcopy +from audit.intakelib.intermediate_representation import ( + replace_range_by_name, +) + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Strip all text fields of leading and trailing whitespace +def convert_to_stripped_string(ir): + new_ir = deepcopy(ir) + for sheet in ir: + # To ensure backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1, we check for both "AdditionalNotes" and "Form" + if sheet["name"] in {"AdditionalNotes", "Form", "Coversheet"}: + for range in sheet["ranges"]: + formatted_values = [ + None if v is None or (not str(v).strip()) else str(v).strip() + for v in range["values"] + ] + new_ir = replace_range_by_name(new_ir, range["name"], formatted_values) + return new_ir diff --git a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py deleted file mode 100644 index 165bdf3667..0000000000 --- a/backend/audit/intakelib/transforms/xform_all_passthrough_id_need_to_be_strings.py +++ /dev/null @@ -1,15 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def all_passthrough_id_need_to_be_strings(ir): - passthrough_ids = get_range_by_name(ir, "passthrough_identifying_number") - new_values = [str(v) if v is not None else None for v in passthrough_ids["values"]] - new_ir = replace_range_by_name(ir, "passthrough_identifying_number", new_values) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py b/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py deleted file mode 100644 index affeaa4d43..0000000000 --- a/backend/audit/intakelib/transforms/xform_eins_need_to_be_strings.py +++ /dev/null @@ -1,14 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def eins_need_to_be_strings(ir): - eins = get_range_by_name(ir, "additional_ein") - new_values = list(map(lambda v: str(v), eins["values"])) - new_ir = replace_range_by_name(ir, "additional_ein", new_values) - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py new file mode 100644 index 0000000000..6e8870ae2e --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_loan_balance_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert end of period loan balance to integers when applicable +def convert_loan_balance_to_integers_or_na(ir): + xform_ir = safe_int_conversion(ir, "loan_balance_at_audit_period_end", {"N/A"}) + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py b/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py deleted file mode 100644 index 6cf05eb0c7..0000000000 --- a/backend/audit/intakelib/transforms/xform_make_sure_notes_to_sefa_are_just_strings.py +++ /dev/null @@ -1,24 +0,0 @@ -import logging -from audit.intakelib.intermediate_representation import ( - get_range_by_name, - replace_range_by_name, -) - -logger = logging.getLogger(__name__) - - -def make_sure_notes_to_sefa_are_just_strings(ir): - note_content = get_range_by_name(ir, "note_content") - note_title = get_range_by_name(ir, "note_title") - new_contents = [] - new_titles = [] - for indx, (note_con, note_tit) in enumerate( - zip(note_content["values"], note_title["values"]) - ): - new_contents.append(str(note_con).strip()) - new_titles.append(str(note_tit).strip()) - - new_ir = replace_range_by_name(ir, "note_content", new_contents) - new_ir = replace_range_by_name(new_ir, "note_title", new_titles) - - return new_ir diff --git a/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py new file mode 100644 index 0000000000..427a522d17 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_number_of_findings_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all number_of_audit_findings to integers +def convert_number_of_findings_to_integers(ir): + xform_ir = safe_int_conversion(ir, "number_of_audit_findings") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_reformat_prior_references.py b/backend/audit/intakelib/transforms/xform_reformat_prior_references.py new file mode 100644 index 0000000000..8b684760eb --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_reformat_prior_references.py @@ -0,0 +1,17 @@ +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, + replace_range_by_name, +) +import re + +logger = logging.getLogger(__name__) + + +def reformat_prior_references(ir): + references = get_range_by_name(ir, "prior_references") + new_values = list( + map(lambda v: re.sub(r"\s+,", ",", str(v).strip()), references["values"]) + ) + new_ir = replace_range_by_name(ir, "prior_references", new_values) + return new_ir diff --git a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py index c9e997eb3c..0397c490fb 100644 --- a/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py +++ b/backend/audit/intakelib/transforms/xform_rename_additional_notes_sheet.py @@ -4,7 +4,8 @@ logger = logging.getLogger(__name__) -# This transform is needed for backwards compatibility with workbook templates 1.0.0 and 1.0.1 +# This transform is needed for backwards compatibility with NotesToSefa workbook 1.0.0 and 1.0.1 +# Once we deprecate those versions, we can remove this transform def rename_additional_notes_sheet_to_form_sheet(ir): new_ir = deepcopy(ir) diff --git a/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py new file mode 100644 index 0000000000..c953ecfb8a --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_subrecipient_amount_need_to_be_integers.py @@ -0,0 +1,11 @@ +import logging +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# Convert all subrecipient_amount to integers +def convert_subrecipient_amount_to_integers(ir): + xform_ir = safe_int_conversion(ir, "subrecipient_amount") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py new file mode 100644 index 0000000000..b3b16d6cb2 --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_total_amount_expended_need_to_be_integers.py @@ -0,0 +1,12 @@ +import logging + +from audit.intakelib.common import safe_int_conversion + +logger = logging.getLogger(__name__) + + +# DESCRIPTION +# convert total amount expended to integers +def convert_total_amount_expended_to_integers(ir): + xform_ir = safe_int_conversion(ir, "total_amount_expended") + return xform_ir diff --git a/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py new file mode 100644 index 0000000000..f782dc132e --- /dev/null +++ b/backend/audit/intakelib/transforms/xform_uniform_cluster_names.py @@ -0,0 +1,27 @@ +import logging +from audit.intakelib.intermediate_representation import ( + get_range_by_name, + replace_range_by_name, +) + +logger = logging.getLogger(__name__) + + +def regenerate_uniform_cluster_names(ir): + state_cluster_names = get_range_by_name(ir, "state_cluster_name") + other_cluster_names = get_range_by_name(ir, "other_cluster_name") + uniform_state_cluster_name_values = [ + None if v is None or not str(v).strip() else str(v).strip().upper() + for v in state_cluster_names["values"] + ] + uniform_other_cluster_name_values = [ + None if v is None or not str(v).strip() else str(v).strip().upper() + for v in other_cluster_names["values"] + ] + new_ir = replace_range_by_name( + ir, "uniform_state_cluster_name", uniform_state_cluster_name_values + ) + xform_ir = replace_range_by_name( + new_ir, "uniform_other_cluster_name", uniform_other_cluster_name_values + ) + return xform_ir diff --git a/backend/cms/templatetags/fac_tags.py b/backend/audit/templatetags/fac_tags.py similarity index 100% rename from backend/cms/templatetags/fac_tags.py rename to backend/audit/templatetags/fac_tags.py diff --git a/backend/audit/test_views.py b/backend/audit/test_views.py index 6853643cd2..6ffc525010 100644 --- a/backend/audit/test_views.py +++ b/backend/audit/test_views.py @@ -153,6 +153,30 @@ def _just_uei_workbooks(uei): return {k: _just_uei(uei, k) for k in workbooks} +class RootPathTests(TestCase): + """ + Do we return the correct response for both authenticated and unauthenticated + users? + """ + + def test_unauthenticated(self): + """Verify the root path returns 200, unauthenticated.""" + result = self.client.get("/") + self.assertEqual(result.status_code, 200) + + def test_authenticated(self): + """Verify the root path redirects, authenticated.""" + user = baker.make(User) + self.client.force_login(user=user) + result = self.client.get("/") + self.assertEqual(result.status_code, 302) + + def test_no_robots(self): + """Verify robots.txt returns 200""" + result = self.client.get("/robots.txt") + self.assertEqual(result.status_code, 200) + + class MySubmissionsViewTests(TestCase): def setUp(self): self.user = baker.make(User) diff --git a/backend/audit/viewlib/__init__.py b/backend/audit/viewlib/__init__.py index a972b9d8a0..f84d2ad60c 100644 --- a/backend/audit/viewlib/__init__.py +++ b/backend/audit/viewlib/__init__.py @@ -1,17 +1,19 @@ +from .home import Home +from .no_robots import no_robots from .submission_progress_view import ( # noqa SubmissionProgressView, submission_progress_check, ) from .tribal_data_consent import TribalDataConsent - from .upload_report_view import UploadReportView - from .unlock_after_certification import UnlockAfterCertificationView # In case we want to iterate through all the views for some reason: views = [ + Home, SubmissionProgressView, TribalDataConsent, UnlockAfterCertificationView, UploadReportView, + no_robots, ] diff --git a/backend/cms/views.py b/backend/audit/viewlib/home.py similarity index 56% rename from backend/cms/views.py rename to backend/audit/viewlib/home.py index 5810372f03..ce30f60ef9 100644 --- a/backend/cms/views.py +++ b/backend/audit/viewlib/home.py @@ -1,12 +1,17 @@ -from .models import Posts from django.views import generic from django.shortcuts import render, redirect from django.urls import reverse -from django.http import HttpResponse # class based views for posts class Home(generic.View): + """ + This is for the root path: / + + It will return the home template if not authenticated and the audit table if + authenticated. + """ + def get(self, request, *args, **kwargs): if request.user.is_authenticated: url = reverse("audit:MySubmissions") @@ -14,17 +19,3 @@ def get(self, request, *args, **kwargs): template_name = "home.html" extra_context = {} return render(request, template_name, extra_context) - - -# class based view for each post -class postdetail(generic.DetailView): - """Basic details class""" - - model = Posts - template_name = "post.html" - - -# robots.txt -def NoRobots(context): - content = "User-agent: *\nDisallow: /" - return HttpResponse(content, content_type="text/plain") diff --git a/backend/audit/viewlib/no_robots.py b/backend/audit/viewlib/no_robots.py new file mode 100644 index 0000000000..151e505b4e --- /dev/null +++ b/backend/audit/viewlib/no_robots.py @@ -0,0 +1,9 @@ +from django.http import HttpResponse + + +def no_robots(_context): + """ + Return Disallow for * + """ + content = "User-agent: *\nDisallow: /" + return HttpResponse(content, content_type="text/plain") diff --git a/backend/audit/views.py b/backend/audit/views.py index dd27d331f2..d50c2ddf95 100644 --- a/backend/audit/views.py +++ b/backend/audit/views.py @@ -64,10 +64,12 @@ validate_secondary_auditors_json, ) from audit.viewlib import ( # noqa + Home, TribalDataConsent, SubmissionProgressView, UnlockAfterCertificationView, UploadReportView, + no_robots, submission_progress_check, ) diff --git a/backend/cms/__init__.py b/backend/cms/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/cms/admin.py b/backend/cms/admin.py deleted file mode 100644 index 78a3e1fb16..0000000000 --- a/backend/cms/admin.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.contrib import admin -from .models import Posts - - -class PostAdmin(admin.ModelAdmin): - """Fields for admin view""" - - fields = ["title", "slug", "author", "meta_description", "content"] - - -admin.site.register(Posts, PostAdmin) diff --git a/backend/cms/apps.py b/backend/cms/apps.py deleted file mode 100644 index 80bf9e5cb6..0000000000 --- a/backend/cms/apps.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.apps import AppConfig - - -class CmsConfig(AppConfig): - default_auto_field = "django.db.models.BigAutoField" - name = "cms" diff --git a/backend/cms/migrations/0001_Create_content_model.py b/backend/cms/migrations/0001_Create_content_model.py deleted file mode 100644 index 596e8bfdca..0000000000 --- a/backend/cms/migrations/0001_Create_content_model.py +++ /dev/null @@ -1,46 +0,0 @@ -# Generated by Django 4.1.2 on 2022-11-02 14:19 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - initial = True - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ] - - operations = [ - migrations.CreateModel( - name="Posts", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("title", models.CharField(max_length=200, unique=True)), - ("slug", models.SlugField(max_length=200, unique=True)), - ("updated_on", models.DateTimeField(auto_now=True)), - ("created_on", models.DateTimeField(auto_now_add=True)), - ("meta_description", models.CharField(max_length=300, null=True)), - ("content", models.TextField()), - ( - "author", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to=settings.AUTH_USER_MODEL, - ), - ), - ], - options={ - "ordering": ["-created_on"], - }, - ), - ] diff --git a/backend/cms/migrations/__init__.py b/backend/cms/migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/cms/models.py b/backend/cms/models.py deleted file mode 100644 index 03eea9f1c7..0000000000 --- a/backend/cms/models.py +++ /dev/null @@ -1,30 +0,0 @@ -# importing django models and users -from django.db import models -from django.contrib.auth.models import User - - -class Posts(models.Model): - """Basic model for posts.""" - - # title field using charfield constraint with unique constraint - title = models.CharField(max_length=200, unique=True) - # slug field auto populated using title with unique constraint - slug = models.SlugField(max_length=200, unique=True) - # author field populated using users database - author = models.ForeignKey(User, on_delete=models.CASCADE) - # and date time fields automatically populated using system time - updated_on = models.DateTimeField(auto_now=True) - created_on = models.DateTimeField(auto_now_add=True) - # content field to store our post - content = models.TextField() - # meta description for SEO benefits and accessibility - meta_description = models.CharField(max_length=300, null=True) - - class Meta: - """Meta information""" - - ordering = ["-created_on"] - - # used while managing models from terminal - def __str__(self): # pragma: no cover - return str(self.title) diff --git a/backend/cms/templatetags/__init__.py b/backend/cms/templatetags/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/cms/tests.py b/backend/cms/tests.py deleted file mode 100644 index d2b618f3fb..0000000000 --- a/backend/cms/tests.py +++ /dev/null @@ -1,4 +0,0 @@ -# from django.test import TestCase - - -# Create your tests here. diff --git a/backend/cms/urls.py b/backend/cms/urls.py deleted file mode 100644 index ea01786eda..0000000000 --- a/backend/cms/urls.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.urls import path -from . import views - - -urlpatterns = [ - # home page - path("", views.Home.as_view(), name="Home"), - # route for posts - # path("/", views.postdetail.as_view(), name="post_detail"), - path("robots.txt", views.NoRobots, name="no_robots"), -] diff --git a/backend/config/settings.py b/backend/config/settings.py index fc3b22591e..16826a310d 100644 --- a/backend/config/settings.py +++ b/backend/config/settings.py @@ -97,6 +97,7 @@ "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", + "django.contrib.humanize", "django.contrib.sessions", "django.contrib.messages", "django.contrib.postgres", @@ -119,7 +120,6 @@ "api", "users", "report_submission", - "cms", # "data_distro", "dissemination", "support", @@ -256,6 +256,7 @@ AWS_S3_PRIVATE_ENDPOINT = os.environ.get( "AWS_S3_PRIVATE_ENDPOINT", "http://minio:9000" ) + AWS_PRIVATE_DEFAULT_ACL = "private" AWS_S3_ENDPOINT_URL = AWS_S3_PRIVATE_ENDPOINT diff --git a/backend/config/urls.py b/backend/config/urls.py index f7ffc17b70..9fe7c0f876 100644 --- a/backend/config/urls.py +++ b/backend/config/urls.py @@ -1,4 +1,5 @@ from api import views +from audit import views as auditviews from django.conf import settings from django.conf.urls.static import static from django.contrib import admin @@ -73,8 +74,9 @@ ), path("audit/", include("audit.urls")), path("dissemination/", include("dissemination.urls")), - # Keep last so we can use short urls for content pages like home page etc. - path("", include("cms.urls")), + # home page & robots.txt + path("", auditviews.Home.as_view(), name="Home"), + path("robots.txt", auditviews.no_robots, name="no_robots"), ] + static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) if settings.ENABLE_DEBUG_TOOLBAR: diff --git a/backend/dissemination/forms.py b/backend/dissemination/forms.py index bc3e41673e..5db043b3ea 100644 --- a/backend/dissemination/forms.py +++ b/backend/dissemination/forms.py @@ -6,6 +6,7 @@ class SearchForm(forms.Form): (x, str(x)) for x in range(2016, 2024) ) # ((2016, "2016"), (2017, "2017"), ..., (2023, "2023")) + # Query params entity_name = forms.CharField(required=False) uei_or_ein = forms.CharField(required=False) aln = forms.CharField(required=False) @@ -14,3 +15,7 @@ class SearchForm(forms.Form): cog_or_oversight = forms.CharField(required=False) agency_name = forms.CharField(required=False) audit_year = forms.MultipleChoiceField(choices=AY_choices, required=False) + + # Display params + limit = forms.CharField(required=False) + page = forms.CharField(required=False) diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py index a0a63a4bbe..2d264fe268 100644 --- a/backend/dissemination/search.py +++ b/backend/dissemination/search.py @@ -49,6 +49,6 @@ def search_general( fiscal_year_match = Q(audit_year__in=audit_years) query.add(fiscal_year_match, Q.AND) - results = General.objects.filter(query) + results = General.objects.filter(query).order_by("-fac_accepted_date") return results diff --git a/backend/dissemination/templates/search.html b/backend/dissemination/templates/search.html index e0308a37c6..0017b70ddf 100644 --- a/backend/dissemination/templates/search.html +++ b/backend/dissemination/templates/search.html @@ -42,14 +42,14 @@

Filters

aria-controls="entity-name">Name (Entity, Auditee, or Auditor) - {% comment %} Fiscal Year {% endcomment %} + {% comment %} Audit Year {% endcomment %}
{% for value, text in form.audit_year.field.choices %} -
+
Filters
+ {% comment %} Hidden page input for use when clicking pagination buttons {% endcomment %} +
-

Search single audit reports

+

Search single audit reports

{% if results %} +
+
+

Sorting

+

+ Use the arrows at the top of each column to sort results. Sorting only applies to the results shown per page. +

+
+
@@ -137,8 +153,8 @@

Search single audit reports

- - + + @@ -146,7 +162,9 @@

Search single audit reports

- + {% comment %} Sorts ascending/descending by the numeric date string (i.e. 20231231) {% endcomment %} +
- Results {{ results|length }} + Results: {{ results_count }} + showing {{ limit }} per page
Acc Date AY Cog or OverViewPDFViewPDF
{{ result.auditee_name }} {{ result.auditee_uei }}{{ result.fac_accepted_date }}{{ result.fac_accepted_date }} + {{ result.audit_year }} {% if result.oversight_agency %} @@ -168,11 +186,13 @@

Search single audit reports

- + @@ -183,70 +203,53 @@

Search single audit reports

- {% else %} -
- an arrow points left, toward the search form -

- Enter your filters and select Search to begin -

+ {% if results.has_previous %} +
  • + + + Previous +
  • + {% endif %} + {% for page_number in results.adjusted_elided_pages %} + {% if page_number == results.paginator.ELLIPSIS %} +
  • + … +
  • + {% else %} +
  • + {{ page_number }} +
  • + {% endif %} + {% endfor %} + {% if results.has_next %} +
  • + + Next + + +
  • + {% endif %} + + + {% else %} +
    + an arrow points left, toward the search form +

    + Enter your filters and select Search to begin +

    +
    + {% endif %}
    - {% endif %} +
    - - -{% endblock %} + +{% endblock content %} diff --git a/backend/dissemination/templates/summary.html b/backend/dissemination/templates/summary.html index 29826ea9f2..ef003d7378 100644 --- a/backend/dissemination/templates/summary.html +++ b/backend/dissemination/templates/summary.html @@ -1,78 +1,424 @@ {% extends "base.html" %} -{% load static %} {% load field_name_to_label %} +{% load getkey %} +{% load humanize %} +{% load sprite_helper %} +{% load static %} {% block content %} -
    +
    -
    - -
    - {% comment %} General Information {% endcomment %} -
    -

    {{ auditee_name }} ({{ auditee_uei }})

    -

    - Report ID:{{ report_id }} + {% comment %} Title & Header {% endcomment %} +

    +
    + Single audit summary + +

    {{ auditee_name }}

    +

    + UEI: {{ auditee_uei }} +

    +
    +
    +

    + Report ID: {{ report_id }}

    - Submission date:{{ general.submitted_date }} + Submission date: {{ general.submitted_date }}

    - Fiscal Year:{{ general.fy_start_date }} to {{ general.fy_end_date }} + Fiscal Year: {{ general.fy_start_date }} to {{ general.fy_end_date }}

    -
    -

    General Information

    -
    -
    - {% for k, v in general.items %} -

    - {{ k|field_name_to_label }}: {{ v }} -

    - {% endfor %} -
    -
    +
    +
    + - {% for section_key, section_list in data.items %} -
    - -

    {{ section_key }}

    - | - {{ section_list|length }} -
    -
    - - - - {% for k in section_list.0 %}{% endfor %} - - - - {% for item in section_list %} - - {% for k, v in item.items %} - {% if v == '' %} - - {% else %} - - {% endif %} - {% endfor %} - - {% endfor %} - -
    {{ k|field_name_to_label }}
    -{{ v }}
    -
    -
    - {% endfor %} -
    +
    +
    +
    +
    + + {% comment %} The grey box of doom {% endcomment %} +
    +
    +
    + {% comment %} Auditee {% endcomment %} +

    Auditee

    + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    + Contact Name: {{ general.auditee_contact_name }} +

    +
    +

    + Contact title: {{ general.auditee_contact_title }} +

    +
    +

    + Email: {{ general.auditee_email }} +

    +
    +

    + Phone: {{ general.auditee_phone }} +

    +
    +

    + Address: {{ general.auditee_address_line_1 }} +

    +
    +

    + City and state: {{ general.auditee_city }}, {{ general.auditee_state }} +

    +
    +

    + Zip code: {{ general.auditee_zip }} +

    +
    +

    + Additional UEIs?  + {% if data|getkey:"Additional UEIs" %} + Y + {% else %} + N + {% endif %} + + {% if not data|getkey:"Additional UEIs" %} + + + Download workbook 6 + + {% else %} + + + Download workbook 6 + + {% endif %} +

    +
    +

    + EIN: {{ general.auditee_ein }} +

    +
    +

    + + Additional EINs?  + {% if data|getkey:"Additional EINs" %} + Y + {% else %} + N + {% endif %} + + {% if not data|getkey:"Additional EINs" %} + + + Download workbook 8 + + {% else %} + + + Download workbook 8 + + {% endif %} +

    +
    +

    + Certifying name: {{ general.auditee_certify_name }} +

    +
    +

    + Certifying title: {{ general.auditee_certify_title }} +

    +
    + + {% comment %} Auditor {% endcomment %} +

    Auditor

    + + + + + + + + + + + + + + + + + + + +
    +

    + Contact Name: {{ general.auditor_contact_name }} +

    +
    +

    + Contact title: {{ general.auditor_contact_title }} +

    +
    +

    + Email: {{ general.auditor_email }} +

    +
    +

    + Phone: {{ general.auditor_phone }} +

    +
    +

    + Address: {{ general.auditor_address_line_1 }} +

    +
    +

    + City and state: {{ general.auditor_city }}, {{ general.auditor_state }} +

    +
    +

    + Zip code: {{ general.auditor_zip }} +

    +
    +

    + Secondary Auditors?  + {% if data|getkey:"Secondary Auditors" %} + Y + {% else %} + N + {% endif %} + + {% if not data|getkey:"Secondary Auditors" %} + + + Download workbook 7 + + {% else %} + + + Download workbook 7 + + {% endif %} +

    +
    + + {% comment %} Summary {% endcomment %} +

    Summary

    + + + + + + + + + {% comment %} Findings text - If none, show the gray box with no link. {% endcomment %} + + + {% comment %} CAP - If none, show the gray box with no link. {% endcomment %} + + + + + +
    +

    + Federal awards:  + {{ data|getkey:"Awards"|length }} + + + + Download workbook 1 + +

    +
    +

    + Notes to SEFA:  + {{ data|getkey:"Notes to SEFA"|length }} + + {% if not data|getkey:"Notes to SEFA" %} + + + Download workbook 2 + + {% else %} + + + Download workbook 2 + + {% endif%} +

    +
    +

    + Findings:  + {{ data|getkey:"Audit Findings"|length }} + + {% if not data|getkey:"Audit Findings" %} + + + Download workbook 3 + + {% else %} + + + Download workbook 3 + + {% endif %} +

    +
    +

    + Findings text:  + {{ data|getkey:"Audit Findings Text"|length }} + + {% if not data|getkey:"Audit Findings Text" %} + + + Download workbook 4 + + {% else %} + + + Download workbook 4 + + {% endif %} +

    +
    +

    + CAP:  + {{ data|getkey:"Corrective Action Plan"|length }} + + {% if not data|getkey:"Corrective Action Plan" %} + + + Download workbook 5 + + {% else %} + + + Download workbook 5 + + {% endif %} +

    +
    +

    + Total federal expenditure:  + + ${{ general.total_amount_expended|intcomma }} + +

    +
    - {% include "audit-metadata.html" %} {% endblock content %} diff --git a/backend/dissemination/templatetags/getkey.py b/backend/dissemination/templatetags/getkey.py new file mode 100644 index 0000000000..39075e0927 --- /dev/null +++ b/backend/dissemination/templatetags/getkey.py @@ -0,0 +1,14 @@ +""" +Custom tag to pull a key with a space out of a dictionary. +Example: +{{ data.'Notes to SEFA' }} does not work. +Instead, {{ data|getkey:"Notes to SEFA" }} +""" +from django import template + +register = template.Library() + + +@register.filter(name="getkey") +def getkey(value, arg): + return value.get(arg, []) diff --git a/backend/dissemination/test_search.py b/backend/dissemination/test_search.py index 013fbaf744..e17e9cc46d 100644 --- a/backend/dissemination/test_search.py +++ b/backend/dissemination/test_search.py @@ -251,5 +251,3 @@ def test_audit_year(self): audit_years=[2020, 2021, 2022], ) self.assertEqual(len(results), 3) - - return diff --git a/backend/dissemination/tests.py b/backend/dissemination/tests.py index 92b72c0807..86607de84e 100644 --- a/backend/dissemination/tests.py +++ b/backend/dissemination/tests.py @@ -13,7 +13,6 @@ from dissemination.models import ( General, FederalAward, - Passthrough, Finding, FindingText, CapText, @@ -124,7 +123,6 @@ def test_summary_context(self): """ baker.make(General, report_id="2022-12-GSAFAC-0000000001", is_public=True) award = baker.make(FederalAward, report_id="2022-12-GSAFAC-0000000001") - passthrough = baker.make(Passthrough, report_id="2022-12-GSAFAC-0000000001") finding = baker.make(Finding, report_id="2022-12-GSAFAC-0000000001") finding_text = baker.make(FindingText, report_id="2022-12-GSAFAC-0000000001") cap_text = baker.make(CapText, report_id="2022-12-GSAFAC-0000000001") @@ -139,10 +137,6 @@ def test_summary_context(self): response.context["data"]["Awards"][0]["additional_award_identification"], award.additional_award_identification, ) - self.assertEquals( - response.context["data"]["Passthrough Entities"][0]["award_reference"], - passthrough.award_reference, - ) self.assertEquals( response.context["data"]["Audit Findings"][0]["reference_number"], finding.reference_number, @@ -158,6 +152,6 @@ def test_summary_context(self): cap_text.contains_chart_or_table, ) self.assertEquals( - response.context["data"]["Notes"][0]["accounting_policies"], + response.context["data"]["Notes to SEFA"][0]["accounting_policies"], note.accounting_policies, ) diff --git a/backend/dissemination/views.py b/backend/dissemination/views.py index 6ed9a7717c..a51e6a15a9 100644 --- a/backend/dissemination/views.py +++ b/backend/dissemination/views.py @@ -1,4 +1,5 @@ -from django.core.exceptions import PermissionDenied +from django.core.exceptions import BadRequest, PermissionDenied +from django.core.paginator import Paginator from django.http import Http404 from django.shortcuts import get_object_or_404, redirect, render from django.views.generic import View @@ -11,11 +12,13 @@ from dissemination.models import ( General, FederalAward, - Passthrough, Finding, FindingText, CapText, Note, + SecondaryAuditor, + AdditionalEin, + AdditionalUei, ) @@ -28,6 +31,7 @@ def get(self, request, *args, **kwargs): def post(self, request, *args, **kwargs): form = SearchForm(request.POST) results = [] + context = {} if form.is_valid(): names = form.cleaned_data["entity_name"].splitlines() @@ -40,6 +44,11 @@ def post(self, request, *args, **kwargs): int(year) for year in form.cleaned_data["audit_year"] ] # Cast strings from HTML to int + # TODO: Add a limit choice field to the form + limit = form.cleaned_data["limit"] or 30 + # Changed in the form via pagination links + page = form.cleaned_data["page"] or 1 + results = search_general( names, uei_or_eins, @@ -49,13 +58,32 @@ def post(self, request, *args, **kwargs): agency_name, audit_years, ) + results_count = results.count() # Total result count + paginator = Paginator( + results, per_page=limit + ) # Paginator object handles results splicing, page count, and pagination buttons + results = paginator.get_page(page) # Results for a given page + results.adjusted_elided_pages = paginator.get_elided_page_range( + page, on_each_side=1 + ) # Pagination buttons, adjust ellipses around the current page + # Reformat these so the date-picker elements in HTML prepopulate if form.cleaned_data["start_date"]: form.cleaned_data["start_date"] = start_date.strftime("%Y-%m-%d") if form.cleaned_data["end_date"]: form.cleaned_data["end_date"] = end_date.strftime("%Y-%m-%d") + else: + raise BadRequest("Form data validation error.", form.errors) + + context = context | { + "form": form, + "limit": limit, + "results": results, + "results_count": results_count, + "page": page, + } - return render(request, "search.html", {"form": form, "results": results}) + return render(request, "search.html", context) class AuditSummaryView(View): @@ -97,11 +125,13 @@ def get_audit_content(self, report_id): further. I.e. remove DB ids or something. """ awards = FederalAward.objects.filter(report_id=report_id) - passthrough_entities = Passthrough.objects.filter(report_id=report_id) audit_findings = Finding.objects.filter(report_id=report_id) audit_findings_text = FindingText.objects.filter(report_id=report_id) corrective_action_plan = CapText.objects.filter(report_id=report_id) notes_to_sefa = Note.objects.filter(report_id=report_id) + secondary_auditors = SecondaryAuditor.objects.filter(report_id=report_id) + additional_ueis = AdditionalUei.objects.filter(report_id=report_id) + additional_eins = AdditionalEin.objects.filter(report_id=report_id) data = {} @@ -109,8 +139,8 @@ def get_audit_content(self, report_id): x for x in awards.values() ] # Take QuerySet to a list of objects - if passthrough_entities.exists(): - data["Passthrough Entities"] = [x for x in passthrough_entities.values()] + if notes_to_sefa.exists(): + data["Notes to SEFA"] = [x for x in notes_to_sefa.values()] if audit_findings.exists(): data["Audit Findings"] = [x for x in audit_findings.values()] if audit_findings_text.exists(): @@ -119,13 +149,12 @@ def get_audit_content(self, report_id): data["Corrective Action Plan"] = [ x for x in corrective_action_plan.values() ] - if notes_to_sefa.exists(): - data["Notes"] = [x for x in notes_to_sefa.values()] - - for key in data: - for item in data[key]: - del item["id"] - del item["report_id"] + if secondary_auditors.exists(): + data["Secondary Auditors"] = [x for x in secondary_auditors.values()] + if additional_ueis.exists(): + data["Additional UEIs"] = [x for x in additional_ueis.values()] + if additional_eins.exists(): + data["Additional EINs"] = [x for x in additional_eins.values()] return data diff --git a/backend/dissemination/workbooklib/additional_eins.py b/backend/dissemination/workbooklib/additional_eins.py index 58eecdc6a2..a0d46316bb 100644 --- a/backend/dissemination/workbooklib/additional_eins.py +++ b/backend/dissemination/workbooklib/additional_eins.py @@ -8,7 +8,6 @@ ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import import openpyxl as pyxl @@ -29,7 +28,6 @@ def generate_additional_eins(dbkey, year, outfile): wb = pyxl.load_workbook(templates["AdditionalEINs"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "additional-eins-workbook") addl_eins = Eins.select().where(Eins.dbkey == g.dbkey) map_simple_columns(wb, mappings, addl_eins) diff --git a/backend/dissemination/workbooklib/additional_ueis.py b/backend/dissemination/workbooklib/additional_ueis.py index 52d3952d0b..98fb382116 100644 --- a/backend/dissemination/workbooklib/additional_ueis.py +++ b/backend/dissemination/workbooklib/additional_ueis.py @@ -8,7 +8,6 @@ ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import import openpyxl as pyxl @@ -28,7 +27,6 @@ def generate_additional_ueis(dbkey, year, outfile): Gen = dynamic_import("Gen", year) wb = pyxl.load_workbook(templates["AdditionalUEIs"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "additional-ueis-workbook") if int(year) >= 22: Ueis = dynamic_import("Ueis", year) addl_ueis = Ueis.select().where(Ueis.dbkey == g.dbkey) diff --git a/backend/dissemination/workbooklib/corrective_action_plan.py b/backend/dissemination/workbooklib/corrective_action_plan.py index 049fa19534..febde0382d 100644 --- a/backend/dissemination/workbooklib/corrective_action_plan.py +++ b/backend/dissemination/workbooklib/corrective_action_plan.py @@ -8,7 +8,6 @@ test_pfix, ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import @@ -33,7 +32,6 @@ def generate_corrective_action_plan(dbkey, year, outfile): ] g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "corrective-action-plan-workbook") captexts = Captext.select().where(Captext.dbkey == g.dbkey) diff --git a/backend/dissemination/workbooklib/excel_creation.py b/backend/dissemination/workbooklib/excel_creation.py index becac8b609..6e2dc21207 100644 --- a/backend/dissemination/workbooklib/excel_creation.py +++ b/backend/dissemination/workbooklib/excel_creation.py @@ -6,7 +6,6 @@ import logging from datetime import date from config import settings -import re import json logger = logging.getLogger(__name__) @@ -211,12 +210,3 @@ def extract_metadata(sheet_json, range): if ("range_name" in scell) and (scell["range_name"] == range): result = scell["formula"] return result - - -def insert_version_and_sheet_name(wb, sheet_json): - ver_cell = extract_metadata(sheet_json, "version") - ver_re = re.search('"(.*?)"', ver_cell)[1] - wb_name_cell = extract_metadata(sheet_json, "section_name") - wb_name_re = re.search('"(.*?)"', wb_name_cell)[1] - set_single_cell_range(wb, "version", ver_re) - set_single_cell_range(wb, "section_name", wb_name_re) diff --git a/backend/dissemination/workbooklib/federal_awards.py b/backend/dissemination/workbooklib/federal_awards.py index fe19b012af..24f6d71e99 100644 --- a/backend/dissemination/workbooklib/federal_awards.py +++ b/backend/dissemination/workbooklib/federal_awards.py @@ -9,7 +9,6 @@ set_range, ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import from config import settings @@ -78,20 +77,32 @@ def int_or_na(o): def _generate_cluster_names(Cfda, cfdas, valid_json): cluster_names = [] + state_cluster_names = [] other_cluster_names = [] cfda: Cfda for cfda in cfdas: if cfda.clustername is None: cluster_names.append("N/A") other_cluster_names.append("") + state_cluster_names.append("") + elif cfda.clustername == "STATE CLUSTER": + cluster_names.append(cfda.clustername) + state_cluster_names.append(cfda.stateclustername) + other_cluster_names.append("") + elif cfda.clustername == "OTHER CLUSTER NOT LISTED ABOVE": + cluster_names.append(cfda.clustername) + other_cluster_names.append(cfda.otherclustername) + state_cluster_names.append("") elif cfda.clustername in valid_json["cluster_names"]: cluster_names.append(cfda.clustername) other_cluster_names.append("") + state_cluster_names.append("") else: logger.debug(f"Cluster {cfda.clustername} not in the list. Replacing.") cluster_names.append("OTHER CLUSTER NOT LISTED ABOVE") other_cluster_names.append(f"{cfda.clustername}") - return (cluster_names, other_cluster_names) + state_cluster_names.append("") + return (cluster_names, other_cluster_names, state_cluster_names) def _fix_addl_award_identification(Cfda, cfdas, dbkey): @@ -190,7 +201,6 @@ def generate_federal_awards(dbkey, year, outfile): # In sheet : in DB g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "federal-awards-workbook") cfdas = Cfda.select().where(Cfda.dbkey == dbkey).order_by(Cfda.index) map_simple_columns(wb, mappings, cfdas) @@ -199,10 +209,8 @@ def generate_federal_awards(dbkey, year, outfile): # they wanted. valid_file = open(f"{settings.BASE_DIR}/schemas/source/base/ClusterNames.json") valid_json = json.load(valid_file) - # This was removed from the CSV... - valid_json["cluster_names"].append("STATE CLUSTER") - (cluster_names, other_cluster_names) = _generate_cluster_names( + (cluster_names, other_cluster_names, state_cluster_names) = _generate_cluster_names( Cfda, cfdas, valid_json ) set_range(wb, "cluster_name", cluster_names) @@ -220,6 +228,20 @@ def generate_federal_awards(dbkey, year, outfile): # We need a `cfda_key` as a magic column for the summation logic to work/be checked. set_range(wb, "cfda_key", full_cfdas, conversion_fun=str) + # We need `uniform_state_cluster_name` and `uniform_other_cluster_name` magic columns for cluster summation logic to work/be checked. + set_range( + wb, + "uniform_state_cluster_name", + [s.strip().upper() for s in state_cluster_names], + conversion_fun=str, + ) + set_range( + wb, + "uniform_other_cluster_name", + [s.strip().upper() for s in other_cluster_names], + conversion_fun=str, + ) + (passthrough_names, passthrough_ids) = _fix_passthroughs( Cfda, Passthrough, cfdas, dbkey ) diff --git a/backend/dissemination/workbooklib/findings.py b/backend/dissemination/workbooklib/findings.py index 453b800739..54e767a030 100644 --- a/backend/dissemination/workbooklib/findings.py +++ b/backend/dissemination/workbooklib/findings.py @@ -7,7 +7,6 @@ set_range, ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import @@ -55,11 +54,40 @@ def sorted_string(s): "N/A", str, ), - # FIXME: We have to calculate, and patch in, is_valid - # is_valid is computed in the workbook ] +def _get_findings_grid(findings_list): + # The original copy of allowed_combos is in audit/intakelib/checks/check_findings_grid_validation.py + allowed_combos = { + "YNNNN", + "YNYNN", + "YNNYN", + "NYNNN", + "NYYNN", + "NYNYN", + "NNYNN", + "NNNYN", + "NNNNY", + } + + attributes = [ + "modifiedopinion", + "othernoncompliance", + "materialweakness", + "significantdeficiency", + "otherfindings", + ] + + return [ + "Y" + if "".join((getattr(finding, attr, "") or "").strip() for attr in attributes) + in allowed_combos + else "N" + for finding in findings_list + ] + + def generate_findings(dbkey, year, outfile): logger.info(f"--- generate findings {dbkey} {year} ---") Gen = dynamic_import("Gen", year) @@ -67,7 +95,6 @@ def generate_findings(dbkey, year, outfile): Cfda = dynamic_import("Cfda", year) wb = pyxl.load_workbook(templates["AuditFindings"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "federal-awards-audit-findings-workbook") cfdas = Cfda.select().where(Cfda.dbkey == g.dbkey).order_by(Cfda.index) # For each of them, I need to generate an elec -> award mapping. @@ -88,6 +115,9 @@ def generate_findings(dbkey, year, outfile): map_simple_columns(wb, mappings, findings) set_range(wb, "award_reference", award_references) + grid = _get_findings_grid(findings) + # We need a magic "is_valid" column, which is computed in the workbook. + set_range(wb, "is_valid", grid, conversion_fun=str) wb.save(outfile) table = generate_dissemination_test_table( diff --git a/backend/dissemination/workbooklib/findings_text.py b/backend/dissemination/workbooklib/findings_text.py index 89a9711093..afbdeeb76a 100644 --- a/backend/dissemination/workbooklib/findings_text.py +++ b/backend/dissemination/workbooklib/findings_text.py @@ -8,7 +8,6 @@ test_pfix, ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import import openpyxl as pyxl @@ -33,7 +32,6 @@ def generate_findings_text(dbkey, year, outfile): wb = pyxl.load_workbook(templates["AuditFindingsText"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "audit-findings-text-workbook") ftexts = Findingstext.select().where(Findingstext.dbkey == g.dbkey) map_simple_columns(wb, mappings, ftexts) diff --git a/backend/dissemination/workbooklib/notes_to_sefa.py b/backend/dissemination/workbooklib/notes_to_sefa.py index fe6c0e6649..ac2cf5eee8 100644 --- a/backend/dissemination/workbooklib/notes_to_sefa.py +++ b/backend/dissemination/workbooklib/notes_to_sefa.py @@ -9,7 +9,6 @@ ) from dissemination.workbooklib.excel_creation import ( - insert_version_and_sheet_name, set_range, ) from dissemination.workbooklib.census_models.census import dynamic_import @@ -47,7 +46,6 @@ def generate_notes_to_sefa(dbkey, year, outfile): wb = pyxl.load_workbook(templates["SEFA"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "notes-to-sefa-workbook") # The mapping is weird. # https://facdissem.census.gov/Documents/DataDownloadKey.xlsx diff --git a/backend/dissemination/workbooklib/secondary_auditors.py b/backend/dissemination/workbooklib/secondary_auditors.py index 090151e146..fadc77711e 100644 --- a/backend/dissemination/workbooklib/secondary_auditors.py +++ b/backend/dissemination/workbooklib/secondary_auditors.py @@ -7,7 +7,6 @@ test_pfix, ) -from dissemination.workbooklib.excel_creation import insert_version_and_sheet_name from dissemination.workbooklib.census_models.census import dynamic_import @@ -58,7 +57,6 @@ def generate_secondary_auditors(dbkey, year, outfile): wb = pyxl.load_workbook(templates["SecondaryAuditors"]) g = set_uei(Gen, wb, dbkey) - insert_version_and_sheet_name(wb, "secondary-auditors-workbook") sec_cpas = Cpas.select().where(Cpas.dbkey == g.dbkey) diff --git a/backend/excel_templates/CorrectiveActionPlanTemplate2019-2022.xlsx b/backend/excel_templates/CorrectiveActionPlanTemplate2019-2022.xlsx deleted file mode 100644 index c336470e9f..0000000000 Binary files a/backend/excel_templates/CorrectiveActionPlanTemplate2019-2022.xlsx and /dev/null differ diff --git a/backend/excel_templates/FederalAwardsExpendedTemplateUG2019.xlsx b/backend/excel_templates/FederalAwardsExpendedTemplateUG2019.xlsx deleted file mode 100644 index c526114687..0000000000 Binary files a/backend/excel_templates/FederalAwardsExpendedTemplateUG2019.xlsx and /dev/null differ diff --git a/backend/excel_templates/FindingsText2019-2022.xlsx b/backend/excel_templates/FindingsText2019-2022.xlsx deleted file mode 100644 index 8e1e52da09..0000000000 Binary files a/backend/excel_templates/FindingsText2019-2022.xlsx and /dev/null differ diff --git a/backend/excel_templates/FindingsUniformGuidanceTemplate2019-2022.xlsx b/backend/excel_templates/FindingsUniformGuidanceTemplate2019-2022.xlsx deleted file mode 100644 index 4b1d285924..0000000000 Binary files a/backend/excel_templates/FindingsUniformGuidanceTemplate2019-2022.xlsx and /dev/null differ diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 97858ef217..672c354c11 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -40,7 +40,6 @@ exclude_dirs = [ "audit/test_validators.py", "node_modules", "dissemination/test_search.py", - "dissemination/tests.py", ] [tool.djlint] diff --git a/backend/report_submission/storages.py b/backend/report_submission/storages.py index 2acb85004d..e3022a8df1 100644 --- a/backend/report_submission/storages.py +++ b/backend/report_submission/storages.py @@ -13,4 +13,5 @@ class S3PrivateStorage(S3Boto3Storage): bucket_name = settings.AWS_PRIVATE_STORAGE_BUCKET_NAME access_key = settings.AWS_PRIVATE_ACCESS_KEY_ID secret_key = settings.AWS_PRIVATE_SECRET_ACCESS_KEY + default_acl = settings.AWS_PRIVATE_DEFAULT_ACL location = "" diff --git a/backend/schemas/output/excel/json/additional-eins-workbook.json b/backend/schemas/output/excel/json/additional-eins-workbook.json index 13e4f0628b..16de065a52 100644 --- a/backend/schemas/output/excel/json/additional-eins-workbook.json +++ b/backend/schemas/output/excel/json/additional-eins-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"AdditionalEins\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "AdditionalEins", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/additional-ueis-workbook.json b/backend/schemas/output/excel/json/additional-ueis-workbook.json index 35be3b3bd5..2e908dbc18 100644 --- a/backend/schemas/output/excel/json/additional-ueis-workbook.json +++ b/backend/schemas/output/excel/json/additional-ueis-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"AdditionalUeis\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "AdditionalUeis", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/audit-findings-text-workbook.json b/backend/schemas/output/excel/json/audit-findings-text-workbook.json index fcbdbb9bf7..b945052611 100644 --- a/backend/schemas/output/excel/json/audit-findings-text-workbook.json +++ b/backend/schemas/output/excel/json/audit-findings-text-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"FindingsText\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "FindingsText", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/corrective-action-plan-workbook.json b/backend/schemas/output/excel/json/corrective-action-plan-workbook.json index 6f94975b68..b7f45277b9 100644 --- a/backend/schemas/output/excel/json/corrective-action-plan-workbook.json +++ b/backend/schemas/output/excel/json/corrective-action-plan-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"CorrectiveActionPlan\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "CorrectiveActionPlan", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json b/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json index e8e0085638..f1ef772e6c 100644 --- a/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json +++ b/backend/schemas/output/excel/json/federal-awards-audit-findings-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"FindingsUniformGuidance\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "FindingsUniformGuidance", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/federal-awards-workbook.json b/backend/schemas/output/excel/json/federal-awards-workbook.json index 12666aee9d..5012ef02f6 100644 --- a/backend/schemas/output/excel/json/federal-awards-workbook.json +++ b/backend/schemas/output/excel/json/federal-awards-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"FederalAwardsExpended\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "FederalAwardsExpended", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/notes-to-sefa-workbook.json b/backend/schemas/output/excel/json/notes-to-sefa-workbook.json index 904f770e5a..d7b22087d8 100644 --- a/backend/schemas/output/excel/json/notes-to-sefa-workbook.json +++ b/backend/schemas/output/excel/json/notes-to-sefa-workbook.json @@ -31,7 +31,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -45,11 +44,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"NotesToSefa\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -63,6 +62,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "NotesToSefa", "width": 48 }, { diff --git a/backend/schemas/output/excel/json/secondary-auditors-workbook.json b/backend/schemas/output/excel/json/secondary-auditors-workbook.json index 224270d9c5..4dda398239 100644 --- a/backend/schemas/output/excel/json/secondary-auditors-workbook.json +++ b/backend/schemas/output/excel/json/secondary-auditors-workbook.json @@ -29,7 +29,6 @@ "single_cells": [ { "format": "text", - "formula": "=\"1.0.2\"", "help": { "link": "https://fac.gov/documentation/validation/#plain_text", "text": "Only plain text is allowed, no emoji, formatting, or other special additions" @@ -43,11 +42,11 @@ "validation": { "type": "NOVALIDATION" }, + "value": "1.0.3", "width": 48 }, { "format": "text", - "formula": "=\"SecondaryAuditors\"", "help": { "link": "https://fac.gov/documentation/validation/#section_name", "text": "The workbook you tried to upload is for a different section." @@ -61,6 +60,7 @@ "validation": { "type": "NOVALIDATION" }, + "value": "SecondaryAuditors", "width": 48 }, { diff --git a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx index 10f50d2b9b..d7976d5900 100644 Binary files a/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx and b/backend/schemas/output/excel/xlsx/additional-eins-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx index ec85bf1fb6..a93d06e528 100644 Binary files a/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx and b/backend/schemas/output/excel/xlsx/additional-ueis-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx index 63a2a61738..d4b99dbba9 100644 Binary files a/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx and b/backend/schemas/output/excel/xlsx/audit-findings-text-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx index 3d514c7306..37183107db 100644 Binary files a/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx and b/backend/schemas/output/excel/xlsx/corrective-action-plan-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx index f7f7b163a9..2651a6b4fe 100644 Binary files a/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx and b/backend/schemas/output/excel/xlsx/federal-awards-audit-findings-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx index 3df883a8ff..1cca51fb80 100644 Binary files a/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx and b/backend/schemas/output/excel/xlsx/federal-awards-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx index 86ba5413a9..df5eebb4bd 100644 Binary files a/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx and b/backend/schemas/output/excel/xlsx/notes-to-sefa-workbook.xlsx differ diff --git a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx index 995f46636d..9bd8ab0986 100644 Binary files a/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx and b/backend/schemas/output/excel/xlsx/secondary-auditors-workbook.xlsx differ diff --git a/backend/schemas/scripts/parse.py b/backend/schemas/scripts/parse.py index 3f48e70c75..83d5870219 100644 --- a/backend/schemas/scripts/parse.py +++ b/backend/schemas/scripts/parse.py @@ -15,13 +15,13 @@ Sheet = NT( "Sheet", - "name single_cells meta_cells open_ranges header_inclusion text_ranges header_height row_height hide_col_from hide_row_from", + "name single_cells meta_cells open_ranges text_ranges header_height row_height hide_col_from hide_row_from", ) Posn = NT( "Posn", "title title_cell range_name range_cell width keep_locked format last_range_cell", ) -SingleCell = NT("SingleCell", "posn validation formula help") +SingleCell = NT("SingleCell", "posn validation formula help value") MetaCell = NT("MetaCell", "posn formula help") MergeableCell = NT("MergeableCell", "start_row end_row start_column end_column") MergedUnreachable = NT("MergedUnreachable", "columns") @@ -85,6 +85,7 @@ def parse_single_cell(spec): parse_validation(get(spec, "validation")), get(spec, "formula"), parse_help(get(spec, "help")), + get(spec, "value", default=None), ) @@ -163,7 +164,7 @@ def parse_text_range(spec): def parse_sheet(spec): # noqa: C901 - sc, mtc, opr, hi, tr = None, None, None, None, None + sc, mtc, opr, tr = None, None, None, None name = get(spec, "name", default="Unnamed Sheet") if "single_cells" in spec: sc = list(map(parse_single_cell, get(spec, "single_cells", default=[]))) @@ -177,10 +178,6 @@ def parse_sheet(spec): # noqa: C901 opr = list(map(parse_open_range, get(spec, "open_ranges", default=[]))) else: opr = [] - if "header_inclusion" in spec: - hi = parse_header_inclusion(get(spec, "header_inclusion")) - else: - hi = HeaderInclusion([]) if "text_ranges" in spec: tr = list(map(parse_text_range, get(spec, "text_ranges", default=[]))) else: @@ -201,7 +198,7 @@ def parse_sheet(spec): # noqa: C901 hrf = get(spec, "hide_row_from") else: hrf = None - return Sheet(name, sc, mtc, opr, hi, tr, hh, rh, hcf, hrf) + return Sheet(name, sc, mtc, opr, tr, hh, rh, hcf, hrf) def parse_spec(spec): diff --git a/backend/schemas/scripts/render.py b/backend/schemas/scripts/render.py index 6ab011968a..5c23b5fe2e 100644 --- a/backend/schemas/scripts/render.py +++ b/backend/schemas/scripts/render.py @@ -26,7 +26,7 @@ "name,column,label_row,range_start_row,range_start,abs_range_start,full_range", ) -MAX_ROWS = 5000 +MAX_ROWS = 10000 XLSX_MAX_ROWS = 1048576 # Excel has a maximum of 1048576 rows XLSX_MAX_COLS = 16384 # Excel has a maximum of 16384 columns @@ -98,8 +98,6 @@ def process_spec(WBNT): unlock_data_entry_cells(WBNT.title_row, ws, sheet) set_column_widths(wb, ws, sheet) set_row_heights(wb, ws, sheet) - if sheet.header_inclusion is not None: - apply_header_cell_style(ws, sheet.header_inclusion) set_wb_security(wb, password) return wb @@ -374,6 +372,10 @@ def process_single_cells(wb, ws, sheet): row = int(o.posn.title_cell[1]) ws.row_dimensions[row].height = sheet.header_height + if o.value: + cell_reference = o.posn.range_cell + ws[cell_reference] = o.value + def process_meta_cells(wb, ws, sheet): print("---- process_meta_cells ----") @@ -429,14 +431,6 @@ def configure_header_cell(ws, r): the_cell.alignment = Alignment(wrapText=True, wrap_text=True) -def apply_header_cell_style(ws, additional_header_cells): - print("---- apply_header_cell_style ----") - print(additional_header_cells) - for ahc in additional_header_cells.cells: - the_cell = ws[ahc] - the_cell.fill = header_row_fill - - def process_text_ranges(wb, ws, sheet): print("---- parse_text_ranges ----") max_width = 72 @@ -462,7 +456,7 @@ def unlock_data_entry_cells(header_row, ws, sheet): for r in sheet.open_ranges: if not r.posn.keep_locked: coords = make_range(r) - for rowndx in range(coords.range_start_row, MAX_ROWS): + for rowndx in range(coords.range_start_row, MAX_ROWS + 1): cell_reference = f"${coords.column}${rowndx}" cell = ws[cell_reference] cell.protection = Protection(locked=False) diff --git a/backend/schemas/source/excel/libs/Sheets.libsonnet b/backend/schemas/source/excel/libs/Sheets.libsonnet index 6562838f0e..187b028c94 100644 --- a/backend/schemas/source/excel/libs/Sheets.libsonnet +++ b/backend/schemas/source/excel/libs/Sheets.libsonnet @@ -41,7 +41,7 @@ local section_names = { }; // All workbooks should get the same version number. -local WORKBOOKS_VERSION = '1.0.2'; +local WORKBOOKS_VERSION = '1.0.3'; { single_cell: single_cell, diff --git a/backend/schemas/source/excel/templates/additional-eins-workbook.jsonnet b/backend/schemas/source/excel/templates/additional-eins-workbook.jsonnet index 8c7424453e..6183ff8232 100644 --- a/backend/schemas/source/excel/templates/additional-eins-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/additional-eins-workbook.jsonnet @@ -31,7 +31,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.ADDITIONAL_EINS + '"', + value: Sheets.section_names.ADDITIONAL_EINS, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/additional-ueis-workbook.jsonnet b/backend/schemas/source/excel/templates/additional-ueis-workbook.jsonnet index 410e3e5600..65ad75e11d 100644 --- a/backend/schemas/source/excel/templates/additional-ueis-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/additional-ueis-workbook.jsonnet @@ -31,7 +31,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.ADDITIONAL_UEIS + '"', + value: Sheets.section_names.ADDITIONAL_UEIS, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/audit-findings-text-workbook.jsonnet b/backend/schemas/source/excel/templates/audit-findings-text-workbook.jsonnet index a891ee49e4..82f075f5dd 100644 --- a/backend/schemas/source/excel/templates/audit-findings-text-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/audit-findings-text-workbook.jsonnet @@ -31,7 +31,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.AUDIT_FINDINGS_TEXT + '"', + value: Sheets.section_names.AUDIT_FINDINGS_TEXT, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/corrective-action-plan-workbook.jsonnet b/backend/schemas/source/excel/templates/corrective-action-plan-workbook.jsonnet index 072ebd3187..931fe75731 100644 --- a/backend/schemas/source/excel/templates/corrective-action-plan-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/corrective-action-plan-workbook.jsonnet @@ -31,7 +31,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.CORRECTIVE_ACTION_PLAN + '"', + value: Sheets.section_names.CORRECTIVE_ACTION_PLAN, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/federal-awards-audit-findings-workbook.jsonnet b/backend/schemas/source/excel/templates/federal-awards-audit-findings-workbook.jsonnet index 9d6982c95b..ca9e86d911 100644 --- a/backend/schemas/source/excel/templates/federal-awards-audit-findings-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/federal-awards-audit-findings-workbook.jsonnet @@ -33,7 +33,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -45,7 +45,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.FEDERAL_AWARDS_AUDIT_FINDINGS + '"', + value: Sheets.section_names.FEDERAL_AWARDS_AUDIT_FINDINGS, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet b/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet index 36d4b5260c..3dc28db39d 100644 --- a/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/federal-awards-workbook.jsonnet @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -55,7 +55,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.FEDERAL_AWARDS + '"', + value: Sheets.section_names.FEDERAL_AWARDS, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/notes-to-sefa-workbook.jsonnet b/backend/schemas/source/excel/templates/notes-to-sefa-workbook.jsonnet index 1a94c9aac7..1899cdb8e7 100644 --- a/backend/schemas/source/excel/templates/notes-to-sefa-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/notes-to-sefa-workbook.jsonnet @@ -32,7 +32,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -44,7 +44,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.NOTES_TO_SEFA + '"', + value: Sheets.section_names.NOTES_TO_SEFA, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/schemas/source/excel/templates/secondary-auditors-workbook.jsonnet b/backend/schemas/source/excel/templates/secondary-auditors-workbook.jsonnet index ab5f44da1b..87f5663e79 100644 --- a/backend/schemas/source/excel/templates/secondary-auditors-workbook.jsonnet +++ b/backend/schemas/source/excel/templates/secondary-auditors-workbook.jsonnet @@ -31,7 +31,7 @@ local single_cells = [ title_cell: 'A2', range_cell: 'B2', format: 'text', - formula: '="' + Sheets.WORKBOOKS_VERSION + '"', + value: Sheets.WORKBOOKS_VERSION, help: Help.plain_text, validation: SV.NoValidation, }, @@ -43,7 +43,7 @@ local single_cells = [ title_cell: 'A3', range_cell: 'B3', format: 'text', - formula: '="' + Sheets.section_names.SECONDARY_AUDITORS + '"', + value: Sheets.section_names.SECONDARY_AUDITORS, help: Help.wrong_workbook_template, validation: SV.NoValidation, }, diff --git a/backend/static/js/search-results-pagination.js b/backend/static/js/search-results-pagination.js new file mode 100644 index 0000000000..9a103df2ba --- /dev/null +++ b/backend/static/js/search-results-pagination.js @@ -0,0 +1,38 @@ +var FORM = document.forms[1]; +const pagination_links = document.querySelectorAll('[aria-label^="Page"]'); +const next_page_link = document.querySelectorAll('[aria-label="Next page"]'); +const previous_page_link = document.querySelectorAll( + '[aria-label="Previous page"]' +); + +function attachEventHandlers() { + // If any pagination links are clicked, set the form element and submit it for a reload + pagination_links.forEach((link) => { + link.addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = link.textContent; + FORM.submit(); + }); + }); + // If the next or previous page buttons are clicked, set the form element to be +/- 1 and submit for a reload + if (next_page_link[0]) { + next_page_link[0].addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = parseInt(FORM.elements['page'].value) + 1; + FORM.submit(); + }); + } + if (previous_page_link[0]) { + previous_page_link[0].addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = parseInt(FORM.elements['page'].value) - 1; + FORM.submit(); + }); + } +} + +function init() { + attachEventHandlers(); +} + +window.onload = init; diff --git a/backend/static/scss/main.scss b/backend/static/scss/main.scss index 7bebb9a9bc..4e3d7c130b 100644 --- a/backend/static/scss/main.scss +++ b/backend/static/scss/main.scss @@ -3,9 +3,11 @@ $theme-font-path: '../fonts', $theme-image-path: '../img', $theme-font-type-sans: 'public-sans', + $theme-font-weight-semibold: 600, $theme-type-scale-md: 7, $theme-type-scale-2xl: 13, $theme-color-success-lighter: #eef8eb, + $theme-color-base-lighter: #efefef, ); @forward 'uswds'; @use '_home.scss'; diff --git a/backend/templates/audit-metadata.html b/backend/templates/audit-metadata.html index 203930b3a4..1797cfb12b 100644 --- a/backend/templates/audit-metadata.html +++ b/backend/templates/audit-metadata.html @@ -1,4 +1,4 @@ - +{% comment %} Audit metadata. Included at the bottom of several screens. {% endcomment %}