diff --git a/backend/audit/cross_validation/naming.py b/backend/audit/cross_validation/naming.py index de31a73966..d3af9cdb3a 100644 --- a/backend/audit/cross_validation/naming.py +++ b/backend/audit/cross_validation/naming.py @@ -1,6 +1,8 @@ from types import new_class from typing import NamedTuple +from audit.models.submission_event import SubmissionEvent + # We need a canonical source of the different versions of each name. @@ -17,6 +19,7 @@ class SectionBabelFish(NamedTuple): snake_case: str # Mostly used for the field names in SingleAuditChecklist. url_tail: str | None # Hyphenated version of snake_case, mostly. workbook_number: int | None # Our upload ordering of workbooks. + submission_event: str # The event type we log to the SubmissionEvents table when this section is updated SECTION_NAMES = { @@ -29,6 +32,7 @@ class SectionBabelFish(NamedTuple): snake_case="additional_eins", url_tail="additional-eins", workbook_number=8, + submission_event=SubmissionEvent.EventType.ADDITIONAL_EINS_UPDATED, ), "additional_ueis": SectionBabelFish( all_caps="ADDITIONAL_UEIS", @@ -39,6 +43,7 @@ class SectionBabelFish(NamedTuple): snake_case="additional_ueis", url_tail="additional-ueis", workbook_number=6, + submission_event=SubmissionEvent.EventType.ADDITIONAL_UEIS_UPDATED, ), "audit_information": SectionBabelFish( all_caps="AUDIT_INFORMATION", @@ -49,6 +54,7 @@ class SectionBabelFish(NamedTuple): snake_case="audit_information", url_tail="audit-information", workbook_number=None, + submission_event=SubmissionEvent.EventType.AUDIT_INFORMATION_UPDATED, ), "corrective_action_plan": SectionBabelFish( all_caps="CORRECTIVE_ACTION_PLAN", @@ -59,6 +65,7 @@ class SectionBabelFish(NamedTuple): reverse_url="report_submission:CAP", url_tail="cap", workbook_number=5, + submission_event=SubmissionEvent.EventType.CORRECTIVE_ACTION_PLAN_UPDATED, ), "federal_awards": SectionBabelFish( all_caps="FEDERAL_AWARDS", @@ -69,6 +76,7 @@ class SectionBabelFish(NamedTuple): snake_case="federal_awards", url_tail="federal-awards", workbook_number=1, + submission_event=SubmissionEvent.EventType.FEDERAL_AWARDS_UPDATED, ), "findings_text": SectionBabelFish( all_caps="FINDINGS_TEXT", @@ -79,6 +87,7 @@ class SectionBabelFish(NamedTuple): snake_case="findings_text", url_tail="audit-findings-text", workbook_number=4, + submission_event=SubmissionEvent.EventType.FEDERAL_AWARDS_AUDIT_FINDINGS_TEXT_UPDATED, ), "findings_uniform_guidance": SectionBabelFish( all_caps="FINDINGS_UNIFORM_GUIDANCE", @@ -89,6 +98,7 @@ class SectionBabelFish(NamedTuple): snake_case="findings_uniform_guidance", url_tail="audit-findings", workbook_number=3, + submission_event=SubmissionEvent.EventType.FINDINGS_UNIFORM_GUIDANCE_UPDATED, ), "general_information": SectionBabelFish( all_caps="GENERAL_INFORMATION", @@ -99,6 +109,7 @@ class SectionBabelFish(NamedTuple): snake_case="general_information", url_tail="general-information", workbook_number=None, + submission_event=SubmissionEvent.EventType.GENERAL_INFORMATION_UPDATED, ), "notes_to_sefa": SectionBabelFish( all_caps="NOTES_TO_SEFA", @@ -109,6 +120,7 @@ class SectionBabelFish(NamedTuple): snake_case="notes_to_sefa", url_tail="notes-to-sefa", workbook_number=2, + submission_event=SubmissionEvent.EventType.NOTES_TO_SEFA_UPDATED, ), "single_audit_report": SectionBabelFish( all_caps="SINGLE_AUDIT_REPORT", @@ -119,6 +131,7 @@ class SectionBabelFish(NamedTuple): snake_case="single_audit_report", url_tail="upload-report", workbook_number=None, + submission_event=SubmissionEvent.EventType.AUDIT_REPORT_PDF_UPDATED, ), "secondary_auditors": SectionBabelFish( all_caps="SECONDARY_AUDITORS", @@ -129,6 +142,7 @@ class SectionBabelFish(NamedTuple): snake_case="secondary_auditors", url_tail="secondary-auditors", workbook_number=7, + submission_event=SubmissionEvent.EventType.SECONDARY_AUDITORS_UPDATED, ), "tribal_data_consent": SectionBabelFish( all_caps="TRIBAL_DATA_CONSENT", @@ -139,6 +153,7 @@ class SectionBabelFish(NamedTuple): snake_case="tribal_data_consent", url_tail=None, workbook_number=None, + submission_event=SubmissionEvent.EventType.TRIBAL_CONSENT_UPDATED, ), } diff --git a/backend/audit/cross_validation/submission_progress_check.py b/backend/audit/cross_validation/submission_progress_check.py index c61f9be124..67a7792374 100644 --- a/backend/audit/cross_validation/submission_progress_check.py +++ b/backend/audit/cross_validation/submission_progress_check.py @@ -1,4 +1,5 @@ from audit.cross_validation.naming import NC, find_section_by_name +from audit.models.submission_event import SubmissionEvent from audit.validators import validate_general_information_complete_json from django.core.exceptions import ValidationError @@ -44,7 +45,7 @@ def submission_progress_check(sac, sar=None, crossval=True): result = {k: None for k in sac["sf_sac_sections"]} for key in sac["sf_sac_sections"]: - result = result | progress_check(sac["sf_sac_sections"], key) + result = result | progress_check(sac, sac["sf_sac_sections"], key) incomplete_sections = [] for k in result: @@ -66,7 +67,7 @@ def submission_progress_check(sac, sar=None, crossval=True): ] -def progress_check(sections, key): +def progress_check(sac, sections, key): """ Given the content of sf_sac_sections from sac_validation_shape (plus a single_audit_report key) and a key, determine whether that key is required, and @@ -111,7 +112,7 @@ def get_num_findings(award): # The General Information has its own condition, as it can be partially completed. if key == "general_information": - return general_information_progress_check(progress, general_info) + return general_information_progress_check(progress, general_info, sac) # If it's not required, it's inactive: if not conditions[key]: @@ -119,12 +120,37 @@ def get_num_findings(award): # If it is required, it should be present if sections.get(key): - return {key: progress | {"display": "complete", "completed": True}} + completed_by, completed_date = section_completed_metadata(sac, key) + + return { + key: progress + | { + "display": "complete", + "completed": True, + "completed_by": completed_by, + "completed_date": completed_date, + } + } return {key: progress | {"display": "incomplete", "completed": False}} -def general_information_progress_check(progress, general_info): +def section_completed_metadata(sac, section_key): + try: + section = find_section_by_name(section_key) + event_type = section.submission_event + + report_id = sac["sf_sac_meta"]["report_id"] + event = SubmissionEvent.objects.filter( + sac__report_id=report_id, event=event_type + ).latest("timestamp") + + return event.user.email, event.timestamp + except SubmissionEvent.DoesNotExist: + return None, None + + +def general_information_progress_check(progress, general_info, sac): """ Given a base "progress" dictionary and the general_info object from a submission, run validations to determine its completeness. Then, return a dictionary with @@ -138,8 +164,18 @@ def general_information_progress_check(progress, general_info): is_general_info_complete = False if is_general_info_complete: + completed_by, completed_date = section_completed_metadata( + sac, "general_information" + ) + return { - "general_information": progress | {"display": "complete", "completed": True} + "general_information": progress + | { + "display": "complete", + "completed": True, + "completed_by": completed_by, + "completed_date": completed_date, + } } return { "general_information": progress | {"display": "incomplete", "completed": False} diff --git a/backend/audit/intake_to_dissemination.py b/backend/audit/intake_to_dissemination.py index a1fc3e490d..ac0290e585 100644 --- a/backend/audit/intake_to_dissemination.py +++ b/backend/audit/intake_to_dissemination.py @@ -25,12 +25,16 @@ def omit(remove, d) -> dict: class IntakeToDissemination(object): - def __init__(self, sac) -> None: + DISSEMINATION = "dissemination" + PRE_CERTIFICATION_REVIEW = "pre_certification_review" + + def __init__(self, sac, mode=DISSEMINATION) -> None: self.single_audit_checklist = sac self.report_id = sac.report_id audit_date = sac.general_information["auditee_fiscal_period_end"] self.audit_year = int(audit_date.split("-")[0]) self.loaded_objects: dict[str, list] = {} + self.mode = mode def load_all(self): load_methods = { @@ -280,18 +284,26 @@ def load_general(self): dates_by_status = self._get_dates_from_sac() status = self.single_audit_checklist.STATUS ready_for_certification_date = dates_by_status[status.READY_FOR_CERTIFICATION] - auditor_certified_date = dates_by_status[status.AUDITOR_CERTIFIED] - auditee_certified_date = dates_by_status[status.AUDITEE_CERTIFIED] - submitted_date = self._convert_utc_to_american_samoa_zone( - dates_by_status[status.SUBMITTED] - ) - fac_accepted_date = submitted_date - auditee_certify_name = auditee_certification["auditee_signature"][ - "auditee_name" - ] - auditee_certify_title = auditee_certification["auditee_signature"][ - "auditee_title" - ] + if self.mode == IntakeToDissemination.DISSEMINATION: + submitted_date = self._convert_utc_to_american_samoa_zone( + dates_by_status[status.SUBMITTED] + ) + fac_accepted_date = submitted_date + auditee_certify_name = auditee_certification["auditee_signature"][ + "auditee_name" + ] + auditee_certify_title = auditee_certification["auditee_signature"][ + "auditee_title" + ] + auditor_certified_date = dates_by_status[status.AUDITOR_CERTIFIED] + auditee_certified_date = dates_by_status[status.AUDITEE_CERTIFIED] + elif self.mode == IntakeToDissemination.PRE_CERTIFICATION_REVIEW: + submitted_date = None + fac_accepted_date = submitted_date + auditee_certify_name = None + auditee_certify_title = None + auditor_certified_date = None + auditee_certified_date = None total_amount_expended = self.single_audit_checklist.federal_awards[ "FederalAwards" diff --git a/backend/audit/templates/audit/auditee-certification-step-1.html b/backend/audit/templates/audit/auditee-certification-step-1.html index 462eee45c6..e5fc5e6cf8 100644 --- a/backend/audit/templates/audit/auditee-certification-step-1.html +++ b/backend/audit/templates/audit/auditee-certification-step-1.html @@ -1,14 +1,43 @@ {% extends "base.html" %} {% load static %} +{% load sprite_helper %} {% block content %}
-
{% csrf_token %}
- Auditee certification checklist + Auditee certification + +

Download and review the SF-SAC and single audit report below. These files are for review only and can't be edited.

+

After reviewing, check the box next to each item to confirm your report meets the requirements.

+ +
+ + +

Download SF-SAC

+ + + + +

Download single audit report

+ +
+

Check the box next to each item confirm your report meets the requirements.

You must check all boxes must be checked in order to certify your single audit.

diff --git a/backend/audit/templates/audit/auditor-certification-step-1.html b/backend/audit/templates/audit/auditor-certification-step-1.html index fa0586e425..afe7763d76 100644 --- a/backend/audit/templates/audit/auditor-certification-step-1.html +++ b/backend/audit/templates/audit/auditor-certification-step-1.html @@ -1,14 +1,43 @@ {% extends "base.html" %} {% load static %} +{% load sprite_helper %} {% block content %}
- {% csrf_token %}
- Auditor certification checklist + Auditor certification + +

Download and review the SF-SAC and single audit report below. These files are for review only and can't be edited.

+

After reviewing, check the box next to each item to confirm your report meets the requirements.

+ +
+ + +

Download SF-SAC

+ + + + +

Download single audit report

+ +
+

Check the box next to each item confirm your report meets the requirements.

You must check all boxes in order to certify your single audit.

@@ -81,8 +110,8 @@

Check the box next to each item confirm your report meets the requirements.<

- Cancel + Cancel
diff --git a/backend/audit/templates/audit/manage-submission.html b/backend/audit/templates/audit/manage-submission.html index 82a704bc32..cf6d2f17ff 100644 --- a/backend/audit/templates/audit/manage-submission.html +++ b/backend/audit/templates/audit/manage-submission.html @@ -71,7 +71,7 @@ {% comment %} Uncomment if/when we chose to keep the flags. -

An asterisk (*) following an email address indicates that no user with this email address has logged in thus far.

+

(*) Indicates user has not logged in to view this submission.

{% endcomment %} str: views.ChangeOrAddRoleView.as_view(), name="ChangeOrAddRoleView", ), + path( + "workbook/xlsx//", + views.PredisseminationXlsxDownloadView.as_view(), + name="PredisseminationXlsxDownload", + ), + path( + "report/pdf/", + views.PredisseminationPdfDownloadView.as_view(), + name="PredisseminationPdfDownload", + ), + path( + "summary-report/xlsx/", + views.PredisseminationSummaryReportDownloadView.as_view(), + name="PredisseminationSummaryReportDownload", + ), ] for form_section in FORM_SECTIONS: diff --git a/backend/audit/views/__init__.py b/backend/audit/views/__init__.py index 6b08bc5636..a334f52911 100644 --- a/backend/audit/views/__init__.py +++ b/backend/audit/views/__init__.py @@ -6,6 +6,11 @@ ChangeAuditorCertifyingOfficialView, ) from .no_robots import no_robots +from .pre_dissemination_download_view import ( + PredisseminationXlsxDownloadView, + PredisseminationPdfDownloadView, + PredisseminationSummaryReportDownloadView, +) from .submission_progress_view import ( # noqa SubmissionProgressView, submission_progress_check, @@ -14,9 +19,9 @@ from .upload_report_view import UploadReportView from .unlock_after_certification import UnlockAfterCertificationView from .views import ( - AuditInfoFormView, AuditeeCertificationStep1View, AuditeeCertificationStep2View, + AuditInfoFormView, AuditorCertificationStep1View, AuditorCertificationStep2View, CertificationView, @@ -31,9 +36,9 @@ # In case we want to iterate through all the views for some reason: views = [ - AuditInfoFormView, AuditeeCertificationStep1View, AuditeeCertificationStep2View, + AuditInfoFormView, AuditorCertificationStep1View, AuditorCertificationStep2View, CertificationView, @@ -44,8 +49,12 @@ EditSubmission, ExcelFileHandlerView, Home, - MySubmissions, ManageSubmissionView, + MySubmissions, + no_robots, + PredisseminationPdfDownloadView, + PredisseminationSummaryReportDownloadView, + PredisseminationXlsxDownloadView, ReadyForCertificationView, SingleAuditReportFileHandlerView, SubmissionProgressView, @@ -53,5 +62,4 @@ TribalDataConsent, UnlockAfterCertificationView, UploadReportView, - no_robots, ] diff --git a/backend/audit/views/pre_dissemination_download_view.py b/backend/audit/views/pre_dissemination_download_view.py new file mode 100644 index 0000000000..b4c39b3bbb --- /dev/null +++ b/backend/audit/views/pre_dissemination_download_view.py @@ -0,0 +1,43 @@ +from django.shortcuts import get_object_or_404, redirect +from django.views.generic import View + +from audit.intake_to_dissemination import IntakeToDissemination +from dissemination.file_downloads import get_download_url, get_filename +from dissemination.summary_reports import generate_presubmission_report +from audit.mixins import ( + SingleAuditChecklistAccessRequiredMixin, +) +from audit.models import SingleAuditChecklist + + +class PredisseminationXlsxDownloadView(SingleAuditChecklistAccessRequiredMixin, View): + def get(self, request, report_id, file_type): + sac = get_object_or_404(SingleAuditChecklist, report_id=report_id) + filename = get_filename(sac, file_type) + + return redirect(get_download_url(filename)) + + +class PredisseminationPdfDownloadView(SingleAuditChecklistAccessRequiredMixin, View): + def get(self, request, report_id): + sac = get_object_or_404(SingleAuditChecklist, report_id=report_id) + filename = get_filename(sac, "report") + + return redirect(get_download_url(filename)) + + +class PredisseminationSummaryReportDownloadView( + SingleAuditChecklistAccessRequiredMixin, View +): + def get(self, request, report_id): + sac = get_object_or_404(SingleAuditChecklist, report_id=report_id) + + intake_to_dissem = IntakeToDissemination( + sac, mode=IntakeToDissemination.PRE_CERTIFICATION_REVIEW + ) + i2d_data = intake_to_dissem.load_all() + + filename = generate_presubmission_report(i2d_data) + download_url = get_download_url(filename) + + return redirect(download_url) diff --git a/backend/audit/views/upload_report_view.py b/backend/audit/views/upload_report_view.py index 52540deaa1..ac5b54e168 100644 --- a/backend/audit/views/upload_report_view.py +++ b/backend/audit/views/upload_report_view.py @@ -43,44 +43,44 @@ def page_number_inputs(self): """ return [ PageInput( - "Financial Statement(s) 2 CFR 200.Sl0(a)", "financial_statements" + "Financial Statement(s) 2 CFR 200.510(a)", "financial_statements" ), PageInput( - "Opinion on Financial Statements 2 CFR 200.SlS(a)", + "Opinion on Financial Statements 2 CFR 200.515(a)", "financial_statements_opinion", ), PageInput( - "Schedule of Expenditures of Federal Awards 2 CFR 200.Sl0(b)", + "Schedule of Expenditures of Federal Awards 2 CFR 200.510(b)", "schedule_expenditures", ), PageInput( - "Opinion or Disclaimer of Opinion on Schedule of Federal Awards 2 CFR 200.SlS(a)", + "Opinion or Disclaimer of Opinion on Schedule of Federal Awards 2 CFR 200.515(a)", "schedule_expenditures_opinion", ), PageInput( - "Uniform Guidance Report on Internal Control 2 CFR 200.SlS(b)", + "Uniform Guidance Report on Internal Control 2 CFR 200.515(b)", "uniform_guidance_control", ), PageInput( - "Uniform Guidance Report on Compliance 2 CFR 200.SlS(c)", + "Uniform Guidance Report on Compliance 2 CFR 200.515(c)", "uniform_guidance_compliance", ), - PageInput("GAS Report on Internal Control 2 CFR 200.SlS(b)", "GAS_control"), + PageInput("GAS Report on Internal Control 2 CFR 200.515(b)", "GAS_control"), PageInput( - "GAS Report on Internal Compliance 2 CFR 200.SlS(b)", "GAS_compliance" + "GAS Report on Internal Compliance 2 CFR 200.515(b)", "GAS_compliance" ), PageInput( - "Schedule of Findings and Questioned Costs 2 CFR 200.SlS(d)", + "Schedule of Findings and Questioned Costs 2 CFR 200.515(d)", "schedule_findings", ), PageInput( - "Summary Schedule of Prior Audit Findings 2 CFR 200.Sll(b)", + "Summary Schedule of Prior Audit Findings 2 CFR 200.511(b)", "schedule_prior_findings", required=False, hint="Only required if prior audit findings exist", ), PageInput( - "Corrective Action Plan (if findings) 2 CFR 200.Sll(c)", + "Corrective Action Plan (if findings) 2 CFR 200.511(c)", "CAP_page", required=False, hint="Only required if findings exist", diff --git a/backend/census_historical_migration/management/commands/csv_to_postgres.py b/backend/census_historical_migration/management/commands/csv_to_postgres.py index 5ff6f47423..d79b7267bd 100644 --- a/backend/census_historical_migration/management/commands/csv_to_postgres.py +++ b/backend/census_historical_migration/management/commands/csv_to_postgres.py @@ -9,6 +9,7 @@ from django.core.management.base import BaseCommand from django.conf import settings from django.apps import apps +from collections import defaultdict logger = logging.getLogger(__name__) logger.setLevel(logging.WARNING) @@ -125,10 +126,11 @@ def get_model_name(self, name): return None def load_data(self, file, model_obj, chunk_size): + dtypes = defaultdict(lambda: str) print("Starting load data to postgres") file.seek(0) rows_loaded = 0 - for df in pd.read_csv(file, iterator=True, chunksize=chunk_size): + for df in pd.read_csv(file, iterator=True, chunksize=chunk_size, dtype=dtypes): # Each row is a dictionary. The columns are the # correct names for our model. So, this should be a # clean way to load the model from a row. diff --git a/backend/census_historical_migration/migrations/0003_remove_migrationerrordetail_error_stack_and_more.py b/backend/census_historical_migration/migrations/0003_remove_migrationerrordetail_error_stack_and_more.py new file mode 100644 index 0000000000..d0392a45af --- /dev/null +++ b/backend/census_historical_migration/migrations/0003_remove_migrationerrordetail_error_stack_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.6 on 2023-12-15 18:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ( + "census_historical_migration", + "0002_reportmigrationstatus_migrationerrordetail", + ), + ] + + operations = [ + migrations.RemoveField( + model_name="migrationerrordetail", + name="error_stack", + ), + migrations.RemoveField( + model_name="migrationerrordetail", + name="error_summary", + ), + migrations.AddField( + model_name="migrationerrordetail", + name="detail", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="migrationerrordetail", + name="exception_class", + field=models.TextField(blank=True, null=True), + ), + migrations.AddField( + model_name="migrationerrordetail", + name="tag", + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/backend/census_historical_migration/models.py b/backend/census_historical_migration/models.py index 99e3502b42..9eb0cd544d 100644 --- a/backend/census_historical_migration/models.py +++ b/backend/census_historical_migration/models.py @@ -457,5 +457,6 @@ class MigrationErrorDetail(models.Model): report_migration_status = models.ForeignKey( ReportMigrationStatus, on_delete=models.CASCADE ) - error_summary = models.CharField(blank=True, null=True) - error_stack = models.TextField() + tag = models.TextField(blank=True, null=True) + exception_class = models.TextField(blank=True, null=True) + detail = models.TextField(blank=True, null=True) diff --git a/backend/audit/file_downloads.py b/backend/dissemination/file_downloads.py similarity index 95% rename from backend/audit/file_downloads.py rename to backend/dissemination/file_downloads.py index e003d77212..9687115be7 100644 --- a/backend/audit/file_downloads.py +++ b/backend/dissemination/file_downloads.py @@ -66,12 +66,14 @@ def get_download_url(filename): ) if file_exists(filename): + # Remove directory information + nicer_filename = filename.split("/")[-1] response = s3_client.generate_presigned_url( ClientMethod="get_object", Params={ "Bucket": settings.AWS_PRIVATE_STORAGE_BUCKET_NAME, "Key": filename, - "ResponseContentDisposition": f"attachment;filename={filename}", + "ResponseContentDisposition": f"attachment;filename={nicer_filename}", }, ExpiresIn=30, ) diff --git a/backend/dissemination/forms.py b/backend/dissemination/forms.py index 9b51cb52c8..917a8fe851 100644 --- a/backend/dissemination/forms.py +++ b/backend/dissemination/forms.py @@ -20,3 +20,5 @@ class SearchForm(forms.Form): # Display params limit = forms.CharField(required=False) page = forms.CharField(required=False) + order_by = forms.CharField(required=False) + order_direction = forms.CharField(required=False) diff --git a/backend/dissemination/management/commands/end_to_end_test_data_generator.py b/backend/dissemination/management/commands/end_to_end_test_data_generator.py index 243496b7c1..a1d442e662 100644 --- a/backend/dissemination/management/commands/end_to_end_test_data_generator.py +++ b/backend/dissemination/management/commands/end_to_end_test_data_generator.py @@ -1,6 +1,7 @@ import os import logging import sys +import argparse from config.settings import ENVIRONMENT from django.core.management.base import BaseCommand @@ -17,11 +18,20 @@ def add_arguments(self, parser): ) parser.add_argument("--dbkeys", type=str, required=False, default="") parser.add_argument("--years", type=str, required=False, default="") + parser.add_argument( + "--store", action=argparse.BooleanOptionalAction, default=False + ) + parser.add_argument( + "--apichecks", action=argparse.BooleanOptionalAction, default=True + ) def handle(self, *args, **options): dbkeys_str = options["dbkeys"] years_str = options["years"] email_str = options["email"] + store_files = options["store"] + run_api_checks = options["apichecks"] + dbkeys = dbkeys_str.split(",") years = years_str.split(",") @@ -50,11 +60,17 @@ def handle(self, *args, **options): f"Generating test reports for DBKEYS: {dbkeys_str} and YEARS: {years_str}" ) for dbkey, year in zip(dbkeys, years): - run_end_to_end(email_str, dbkey, year) + run_end_to_end(email_str, dbkey, year, store_files, run_api_checks) else: for pair in defaults: logger.info("Running {}-{} end-to-end".format(pair[0], pair[1])) - run_end_to_end(email_str, str(pair[0]), str(pair[1])) + run_end_to_end( + email_str, + str(pair[0]), + str(pair[1]), + store_files, + run_api_checks, + ) else: logger.error( "Cannot run end-to-end workbook generation in production. Exiting." diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py index 6a6d209bf5..4a59a4a0e8 100644 --- a/backend/dissemination/search.py +++ b/backend/dissemination/search.py @@ -1,6 +1,25 @@ from django.db.models import Q +from collections import namedtuple as NT +from dissemination.models import General, FederalAward +import logging -from dissemination.models import General, FederalAward, Finding +logger = logging.getLogger(__name__) +ALN = NT("ALN", "prefix, program") + + +class ORDER_BY: + fac_accepted_date = "fac_accepted_date" + auditee_name = "auditee_name" + auditee_uei = "auditee_uei" + audit_year = "audit_year" + cog_over = "cog_over" + findings_my_aln = "findings_my_aln" + findings_all_aln = "findings_all_aln" + + +class DIRECTION: + ascending = "ascending" + descending = "descending" def search_general( @@ -14,13 +33,28 @@ def search_general( audit_years=None, auditee_state=None, include_private=False, + order_by=ORDER_BY.fac_accepted_date, + order_direction=DIRECTION.ascending, ): - query = Q() + if not order_by: + order_by = ORDER_BY.fac_accepted_date + if not order_direction: + order_direction = DIRECTION.ascending + + query = _initialize_query(include_private) - # 'alns' gets processed before the match query function, as they get used again after the main search. + split_alns = None + agency_numbers = None if alns: split_alns, agency_numbers = _split_alns(alns) - query.add(_get_aln_match_query(split_alns, agency_numbers), Q.AND) + query_set = _get_aln_match_query(split_alns, agency_numbers) + # If we did a search on ALNs, and got nothing (because it does not exist), + # we need to bail out from the entire search early with no results. + if not query_set: + return [] + else: + # If results came back from our ALN query, add it to the Q() and continue. + query.add(query_set, Q.AND) query.add(_get_names_match_query(names), Q.AND) query.add(_get_uei_or_eins_match_query(uei_or_eins), Q.AND) @@ -30,15 +64,72 @@ def search_general( query.add(_get_audit_years_match_query(audit_years), Q.AND) query.add(_get_auditee_state_match_query(auditee_state), Q.AND) - if not include_private: - query.add(Q(is_public=True), Q.AND) - - results = General.objects.filter(query).order_by("-fac_accepted_date") - + # Create the queryset. It's lazy, so it doesn't hit the database yet. + results = General.objects.filter(query) + # Attach a sort field and direction. + results = _sort_results(results, order_direction, order_by) + # We can apply a "limit", which is a slice + # https://docs.djangoproject.com/en/4.2/topics/db/queries/#limiting-querysets + # results = results[:1000] + + # Accessing the results object will run the query. + # We want to attach bonus ALN fields after getting results. + # Running order_by on the same queryset will hit the databse again, which will wipe our custom fields. + # So, if we want to sort by the ALN fields, we need to do it locally and after the _sort_results function. if alns: results = _attach_finding_my_aln_and_finding_all_aln_fields( results, split_alns, agency_numbers ) + if order_by == ORDER_BY.findings_my_aln: + results = sorted( + results, + key=lambda obj: obj.finding_my_aln, + reverse=bool(order_direction == DIRECTION.descending), + ) + elif order_by == ORDER_BY.findings_all_aln: + results = sorted( + results, + key=lambda obj: obj.finding_all_aln, + reverse=bool(order_direction == DIRECTION.descending), + ) + + return results + + +def _initialize_query(include_private: bool): + query = Q() + # Tribal access limiter. + if not include_private: + query.add(Q(is_public=True), Q.AND) + return query + + +def _sort_results(results, order_direction, order_by): + # Instead of nesting conditions, we'll prep a string + # for determining the sort direction. + match order_direction: + case DIRECTION.ascending: + direction = "" + case _: + direction = "-" + + # Now, apply the sort that we pass in front the front-end. + match order_by: + case ORDER_BY.auditee_name: + results = results.order_by(f"{direction}auditee_name") + case ORDER_BY.auditee_uei: + results = results.order_by(f"{direction}auditee_uei") + case ORDER_BY.fac_accepted_date: + results = results.order_by(f"{direction}fac_accepted_date") + case ORDER_BY.audit_year: + results = results.order_by(f"{direction}audit_year") + case ORDER_BY.cog_over: + if order_direction == DIRECTION.ascending: + results = results.order_by("cognizant_agency") + else: + results = results.order_by("oversight_agency") + case _: + results = results.order_by(f"{direction}fac_accepted_date") return results @@ -61,8 +152,8 @@ def _split_alns(alns): if len(split_aln) == 2: # The [wrapping] is so the tuple goes into the set as a tuple. # Otherwise, the individual elements go in unpaired. - split_alns.update([tuple(split_aln)]) - + # split_alns.update([tuple(split_aln)]) + split_alns.update([ALN(split_aln[0], split_aln[1])]) return split_alns, agency_numbers @@ -73,9 +164,9 @@ def _get_aln_report_ids(split_alns, agency_numbers): """ report_ids = set() # Matching on a specific ALN, such as '12.345' - for aln_list in split_alns: + for aln in split_alns: matching_awards = FederalAward.objects.filter( - federal_agency_prefix=aln_list[0], federal_award_extension=aln_list[1] + federal_agency_prefix=aln.prefix, federal_award_extension=aln.program ).values() if matching_awards: for matching_award in matching_awards: @@ -99,31 +190,62 @@ def _attach_finding_my_aln_and_finding_all_aln_fields( ): """ Given the results QuerySet (full of 'General' objects) and an ALN query string, - return the modified QuerySet, where each 'General' object has two new fields. + return a list of 'General' objects, where each object has two new fields. The process: - 1. Get findings that fall under the given reports. - 2. For each finding, get the relevant award. This is to access its ALN. - 3. For each finding/award pair, they are either: - a. Under one of my ALNs, so we update finding_my_aln to True. - b. Under any other ALN, so we update finding_all_aln to True. + 1. Pull the results report_ids into a list. + 2. Construct queries for my_aln vs all_aln + a. aln_q - All awards with findings under the given ALNs + b. not_aln_q - All awards with findings under NOT the given ALNs + 2. Make the two queries for FederalAwards, 'finding_on_my_alns' and 'finding_on_all_alns'. Ensure results are under the given report_ids. + 3. For every General under results, we check: + a. If a FederalAward in 'finding_on_my_alns' has a report_id under this General, finding_my_aln = True + b. If a FederalAward in 'finding_all_my_alns' has a report_id under this General, finding_all_aln = True + 4. Return the updated results QuerySet. """ - for result in results: - result.finding_my_aln = False - result.finding_all_aln = False - matching_findings = Finding.objects.filter(report_id=result.report_id) - - for finding in matching_findings: - matching_award = FederalAward.objects.get( - report_id=result.report_id, award_reference=finding.award_reference - ) - prefix = matching_award.federal_agency_prefix - extension = matching_award.federal_award_extension - - if ((prefix, extension) in split_alns) or (prefix in agency_numbers): - result.finding_my_aln = True - else: - result.finding_all_aln = True + report_ids = list(results.values_list("report_id", flat=True)) + + aln_q = Q() + for aln in split_alns: + aln_q.add( + Q(federal_agency_prefix=aln.prefix) + & Q(federal_award_extension=aln.program), + Q.OR, + ) + for agency in agency_numbers: + aln_q.add(Q(federal_agency_prefix=agency), Q.OR) + + not_aln_q = Q() + for aln in split_alns: + not_aln_q.add( + ~( + Q(federal_agency_prefix=aln.prefix) + & Q(federal_award_extension=aln.program) + ), + Q.AND, + ) + not_aln_q.add(~Q(federal_agency_prefix__in=list(agency_numbers)), Q.AND) + + finding_on_my_alns = FederalAward.objects.filter( + aln_q, + report_id__in=report_ids, + findings_count__gt=0, + ) + finding_on_any_aln = FederalAward.objects.filter( + not_aln_q, + report_id__in=report_ids, + findings_count__gt=0, + ) + + for general in results: + general.finding_my_aln = False + general.finding_all_aln = False + for relevant_award in finding_on_my_alns: + if relevant_award.report_id == general.report_id: + general.finding_my_aln = True + for relevant_award in finding_on_any_aln: + if relevant_award.report_id == general.report_id: + general.finding_all_aln = True return results diff --git a/backend/dissemination/summary_reports.py b/backend/dissemination/summary_reports.py new file mode 100644 index 0000000000..b80b9c32de --- /dev/null +++ b/backend/dissemination/summary_reports.py @@ -0,0 +1,476 @@ +from datetime import datetime +import openpyxl as pyxl +import io +import logging +import uuid + +from boto3 import client as boto3_client +from botocore.client import ClientError, Config + +from django.conf import settings +from fs.memoryfs import MemoryFS + +from openpyxl.workbook.defined_name import DefinedName +from openpyxl.utils import quote_sheetname + +from dissemination.models import ( + AdditionalEin, + AdditionalUei, + CapText, + FederalAward, + Finding, + FindingText, + General, + Note, + Passthrough, + SecondaryAuditor, +) + +logger = logging.getLogger(__name__) + +models = [ + AdditionalEin, + AdditionalUei, + CapText, + FederalAward, + Finding, + FindingText, + General, + Note, + Passthrough, + SecondaryAuditor, +] + + +columns = [ + "A", + "B", + "C", + "D", + "E", + "F", + "G", + "H", + "I", + "J", + "K", + "L", + "M", + "N", + "O", + "P", + "Q", + "R", + "S", + "T", + "U", + "V", + "W", + "X", + "Y", + "Z", + "AA", + "AB", + "AC", + "AD", + "AE", + "AF", + "AG", + "AH", + "AI", + "AJ", + "AK", + "AL", + "AM", + "AN", + "AO", + "AP", + "AQ", + "AR", + "AS", + "AT", + "AU", + "AV", + "AW", + "AX", + "AY", + "AZ", + "BA", + "BB", + "BC", + "BD", + "BE", + "BF", + "BG", + "BH", + "BI", + "BJ", +] + +field_name_ordered = { + "general": [ + "report_id", + "audit_year", + "total_amount_expended", + "entity_type", + "fy_start_date", + "fy_end_date", + "audit_type", + "audit_period_covered", + "number_months", + "auditee_uei", + "auditee_ein", + "auditee_name", + "auditee_address_line_1", + "auditee_city", + "auditee_state", + "auditee_zip", + "auditee_contact_name", + "auditee_contact_title", + "auditee_phone", + "auditee_email", + "auditee_certified_date", + "auditee_certify_name", + "auditee_certify_title", + "auditor_ein", + "auditor_firm_name", + "auditor_address_line_1", + "auditor_city", + "auditor_state", + "auditor_zip", + "auditor_country", + "auditor_contact_name", + "auditor_contact_title", + "auditor_phone", + "auditor_email", + "auditor_foreign_address", + "auditor_certified_date", + "cognizant_agency", + "oversight_agency", + "type_audit_code", + "sp_framework_basis", + "is_sp_framework_required", + "is_going_concern_included", + "is_internal_control_deficiency_disclosed", + "is_internal_control_material_weakness_disclosed", + "is_material_noncompliance_disclosed", + "gaap_results", + "is_aicpa_audit_guide_included", + "sp_framework_opinions", + "agencies_with_prior_findings", + "dollar_threshold", + "is_low_risk_auditee", + "is_additional_ueis", + "date_created", + "fac_accepted_date", + "ready_for_certification_date", + "submitted_date", + "data_source", + "is_public", + ], + "federalaward": [ + "report_id", + "award_reference", + "federal_agency_prefix", + "federal_award_extension", + "findings_count", + "additional_award_identification", + "federal_program_name", + "amount_expended", + "federal_program_total", + "cluster_name", + "state_cluster_name", + "other_cluster_name", + "cluster_total", + "is_direct", + "is_passthrough_award", + "passthrough_amount", + "is_major", + "audit_report_type", + "is_loan", + "loan_balance", + ], + "finding": [ + "report_id", + "award_reference", + "reference_number", + "type_requirement", + "is_modified_opinion", + "is_other_findings", + "is_material_weakness", + "is_significant_deficiency", + "is_other_matters", + "is_questioned_costs", + "is_repeat_finding", + "prior_finding_ref_numbers", + ], + "findingtext": [ + "id", + "report_id", + "finding_ref_number", + "contains_chart_or_table", + "finding_text", + ], + "note": [ + "id", + "report_id", + "note_title", + "is_minimis_rate_used", + "accounting_policies", + "rate_explained", + "content", + "contains_chart_or_table", + ], + "captext": [ + "report_id", + "finding_ref_number", + "planned_action", + "contains_chart_or_table", + ], + "additionalein": ["report_id", "additional_ein"], + "additionaluei": ["report_id", "additional_uei"], + "passthrough": [ + "report_id", + "award_reference", + "passthrough_name", + "passthrough_id", + ], + "secondaryauditor": [ + "report_id", + "auditor_name", + "auditor_ein", + "address_street", + "address_city", + "address_state", + "address_zipcode", + "contact_name", + "contact_title", + "contact_email", + "contact_phone", + ], +} + + +def set_column_widths(worksheet): + dims = {} + for row in worksheet.rows: + for cell in row: + if cell.value: + dims[cell.column] = max( + (dims.get(cell.column, 0), len(str(cell.value))) + ) + for col, value in dims.items(): + # Pad the column by a bit, so things are not cramped. + worksheet.column_dimensions[columns[col - 1]].width = int(value * 1.2) + + +def protect_sheet(sheet): + sheet.protection.sheet = True + sheet.protection.password = str(uuid.uuid4()) + sheet.protection.enable() + + +def insert_precert_coversheet(workbook): + sheet = workbook.create_sheet("Coversheet", 0) + sheet.append(["Time created", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")]) + sheet.append(["Note", "This file is for review only and can't be edited."]) + set_column_widths(sheet) + protect_sheet(sheet) + + +def insert_dissem_coversheet(workbook): + sheet = workbook.create_sheet("Coversheet", 0) + sheet.append(["Time created", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")]) + set_column_widths(sheet) + + +def gather_report_data_dissemination(report_ids): + """ + Given a set of report IDs, fetch the disseminated data for each and asssemble into a dictionary with the following shape: + + { + general: { + field_names: [], + entries: [], + }, + federal_award: { + field_names: [], + entries: [], + }, + ... + } + """ + + # Make report IDs unique + report_ids = set(report_ids) + + data = {} + + for model in models: + model_name = model.__name__.lower() + + # This pulls directly from the model + # fields = model._meta.get_fields() + # field_names = [f.name for f in fields] + # This uses the ordered columns above + field_names = field_name_ordered[model_name] + + data[model_name] = {"field_names": field_names, "entries": []} + + for report_id in report_ids: + objects = model.objects.all().filter(report_id=report_id) + for obj in objects: + data[model_name]["entries"].append( + [getattr(obj, field_name) for field_name in field_names] + ) + return data + + +def gather_report_data_pre_certification(i2d_data): + """ + Given a sac object dict, asssemble into a dictionary with the following shape: + + { + general: { + field_names: [], + entries: [], + }, + federal_award: { + field_names: [], + entries: [], + }, + ... + } + """ + + # Map IntakeToDissemination names to the dissemination table names + i2d_to_dissemination = { + "Generals": General, + "SecondaryAuditors": SecondaryAuditor, + "FederalAwards": FederalAward, + "Findings": Finding, + "FindingTexts": FindingText, + "Passthroughs": Passthrough, + "CapTexts": CapText, + "Notes": Note, + "AdditionalUEIs": AdditionalUei, + "AdditionalEINs": AdditionalEin, + } + + # Move the IntakeToDissemination data to dissemination_data, under the proper naming scheme. + dissemination_data = {} + for name_i2d, model in i2d_to_dissemination.items(): + dissemination_data[model.__name__.lower()] = i2d_data.get(name_i2d) + + data = {} + + # For every model (FederalAward, CapText, etc), add the skeleton object ('field_names' and empty 'entries') to 'data'. + # Then, for every object in the dissemination_data (objects of type FederalAward, CapText, etc) create a row (array) for the summary. + # Every row is created by looping over the field names and appending the data. + # We also strip tzinfo from the dates, because excel doesn't like them. + # Once a row is created, append it to the data[ModelName]['entries'] array. + for model in models: + model_name = model.__name__.lower() + # This pulls directly from the model + # fields = model._meta.get_fields() + # field_names = [f.name for f in fields] + # This uses the ordered columns above + field_names = field_name_ordered[model_name] + data[model_name] = {"field_names": field_names, "entries": []} + + for obj in dissemination_data[model_name]: + row = [] + for field_name in field_names: + value = getattr(obj, field_name) + # Wipe tzinfo + if isinstance(value, datetime): + value = value.replace(tzinfo=None) + row.append(value) + data[model_name]["entries"].append(row) + + return data + + +def create_workbook(data, protect_sheets=False): + workbook = pyxl.Workbook() + + for sheet_name in data.keys(): + sheet = workbook.create_sheet(sheet_name) + + # create a header row with the field names + sheet.append(data[sheet_name]["field_names"]) + + # append a new row for each entry in the dataset + for entry in data[sheet_name]["entries"]: + sheet.append(entry) + + # add named ranges for the columns, now that the data is loaded. + for index, field_name in enumerate(data[sheet_name]["field_names"]): + coordinate = f"${columns[index]}$2:${columns[index]}${2 + len(data[sheet_name]['entries'])}" + ref = f"{quote_sheetname(sheet.title)}!{coordinate}" + named_range = DefinedName(f"{sheet_name}_{field_name}", attr_text=ref) + workbook.defined_names.add(named_range) + + set_column_widths(sheet) + if protect_sheets: + protect_sheet(sheet) + + # remove sheet that is created during workbook construction + workbook.remove_sheet(workbook.get_sheet_by_name("Sheet")) + + return workbook + + +def persist_workbook(workbook): + s3_client = boto3_client( + service_name="s3", + region_name=settings.AWS_S3_PRIVATE_REGION_NAME, + aws_access_key_id=settings.AWS_PRIVATE_ACCESS_KEY_ID, + aws_secret_access_key=settings.AWS_PRIVATE_SECRET_ACCESS_KEY, + endpoint_url=settings.AWS_S3_PRIVATE_INTERNAL_ENDPOINT, + config=Config(signature_version="s3v4"), + ) + + with MemoryFS() as mem_fs: + now = datetime.utcnow().strftime("%Y%m%d%H%M%S") + filename = f"fac-summary-report-{now}.xlsx" + s3_dir = "temp" + + workbook_write_fp = mem_fs.openbin(filename, mode="w") + workbook.save(workbook_write_fp) + workbook_read_fp = mem_fs.openbin(filename, mode="r") + workbook_read_fp.seek(0) + content = workbook_read_fp.read() + workbook_bytes = io.BytesIO(content) + + try: + s3_client.put_object( + Body=workbook_bytes, + Bucket=settings.AWS_PRIVATE_STORAGE_BUCKET_NAME, + Key=f"{s3_dir}/{filename}", + ) + except ClientError: + logger.warn(f"Unable to put summary report file {filename} in S3!") + raise + + return f"temp/{filename}" + + +def generate_summary_report(report_ids): + data = gather_report_data_dissemination(report_ids) + workbook = create_workbook(data) + insert_dissem_coversheet(workbook) + filename = persist_workbook(workbook) + + return filename + + +def generate_presubmission_report(i2d_data): + data = gather_report_data_pre_certification(i2d_data) + workbook = create_workbook(data, protect_sheets=True) + insert_precert_coversheet(workbook) + workbook.security.workbookPassword = str(uuid.uuid4()) + workbook.security.lockStructure = True + filename = persist_workbook(workbook) + + return filename diff --git a/backend/dissemination/templates/search-table-header.html b/backend/dissemination/templates/search-table-header.html new file mode 100644 index 0000000000..755a7376b3 --- /dev/null +++ b/backend/dissemination/templates/search-table-header.html @@ -0,0 +1,28 @@ +{% load sprite_helper %} + +
+

+ {{ friendly_title }} +

+ +
+ diff --git a/backend/dissemination/templates/search.html b/backend/dissemination/templates/search.html index f72c824e42..e4b628cc9a 100644 --- a/backend/dissemination/templates/search.html +++ b/backend/dissemination/templates/search.html @@ -2,9 +2,9 @@ {% load static %} {% load sprite_helper %} {% block content %} -
+
-
+

Filters

Filters name="agency_name" value="{{ form.cleaned_data.agency_name }}" />
-
+ {% comment %} State {% endcomment %} + +
+ id="page" + name="page" + type="number" + value="{{ page }}" + hidden /> + + {% comment %} Hidden order and direction inputs for use when clicking a sort button in the table header {% endcomment %} + +
-
+

Search single audit reports

- {% if results %} + {% if results|length > 0 %}

Sorting

- Use the arrows at the top of each column to sort results. Sorting only applies to the results shown per page. + Use the arrows at the top of each column to sort results.

@@ -165,19 +183,19 @@

Sorting

showing {{ limit }} per page

- +
- - - - - + {% include "search-table-header.html" with friendly_title="Name" field_name="auditee_name"%} + {% include "search-table-header.html" with friendly_title="UEI or EIN" field_name="auditee_uei"%} + {% include "search-table-header.html" with friendly_title="Acc Date" field_name="fac_accepted_date"%} + {% include "search-table-header.html" with friendly_title="AY" field_name="audit_year"%} + {% include "search-table-header.html" with friendly_title="Cog or Over" field_name="cog_over"%} {% if results.0.finding_my_aln is not None%} - - + {% include "search-table-header.html" with friendly_title="Finding my ALN" field_name="findings_my_aln"%} + {% include "search-table-header.html" with friendly_title="Finding all ALN" field_name="findings_all_aln"%} {% endif %} @@ -284,18 +302,34 @@

Sorting

{% endif %} - {% else %} -
-
-

Searching the FAC database

-

- Learn more about how our search filters work on our Search Resources page. -

+ {% elif results is not None %} +
+
+

Searching the FAC database

+

+ Learn more about how our search filters work on our Search Resources page. +

+
-
an arrow points left, toward the search form +

+ No results found. +

+
+ {% else %} +
+
+

Searching the FAC database

+

+ Learn more about how our search filters work on our Search Resources page. +

+
+
+
+ an arrow points left, toward the search form

Enter your filters and select Search to begin

@@ -329,6 +363,6 @@

Please rota id="orientation-toggle" aria-controls="device-orientation-modal" data-open-modal> - + {% endblock content %} diff --git a/backend/dissemination/templates/summary.html b/backend/dissemination/templates/summary.html index ef003d7378..d8225c4752 100644 --- a/backend/dissemination/templates/summary.html +++ b/backend/dissemination/templates/summary.html @@ -31,7 +31,18 @@

- + +

SF-SAC

+ + + ", + views.SummaryReportDownloadView.as_view(), + name="SummaryReportDownload", + ), path("search/", views.Search.as_view(), name="Search"), path("summary/", views.AuditSummaryView.as_view(), name="Summary"), ] diff --git a/backend/dissemination/views.py b/backend/dissemination/views.py index 8d2bab9bf9..75c722a6f6 100644 --- a/backend/dissemination/views.py +++ b/backend/dissemination/views.py @@ -6,11 +6,11 @@ from django.shortcuts import get_object_or_404, redirect, render from django.views.generic import View -from audit.file_downloads import get_download_url, get_filename from audit.models import SingleAuditChecklist from config.settings import STATE_ABBREVS +from dissemination.file_downloads import get_download_url, get_filename from dissemination.forms import SearchForm from dissemination.search import search_general from dissemination.mixins import ReportAccessRequiredMixin @@ -25,6 +25,7 @@ AdditionalEin, AdditionalUei, ) +from dissemination.summary_reports import generate_summary_report from users.permissions import can_read_tribal @@ -69,6 +70,8 @@ def post(self, request, *args, **kwargs): int(year) for year in form.cleaned_data["audit_year"] ] # Cast strings from HTML to int auditee_state = form.cleaned_data["auditee_state"] + order_by = form.cleaned_data["order_by"] + order_direction = form.cleaned_data["order_direction"] # TODO: Add a limit choice field to the form limit = form.cleaned_data["limit"] or 30 @@ -89,8 +92,10 @@ def post(self, request, *args, **kwargs): audit_years=audit_years, auditee_state=auditee_state, include_private=include_private, + order_by=order_by, + order_direction=order_direction, ) - results_count = results.count() + results_count = len(results) # Reset page to one if the page number surpasses how many pages there actually are if page > math.ceil(results_count / limit): page = 1 @@ -109,6 +114,9 @@ def post(self, request, *args, **kwargs): else: raise BadRequest("Form data validation error.", form.errors) + if order_by is None: + order_by = "acceptance_date" + context = context | { "form": form, "state_abbrevs": STATE_ABBREVS, @@ -116,6 +124,8 @@ def post(self, request, *args, **kwargs): "results": results, "results_count": results_count, "page": page, + "order_by": order_by, + "order_direction": order_direction, } return render(request, "search.html", context) @@ -213,3 +223,12 @@ def get(self, request, report_id, file_type): filename = get_filename(sac, file_type) return redirect(get_download_url(filename)) + + +class SummaryReportDownloadView(ReportAccessRequiredMixin, View): + def get(self, request, report_id): + sac = get_object_or_404(General, report_id=report_id) + filename = generate_summary_report([sac.report_id]) + download_url = get_download_url(filename) + + return redirect(download_url) diff --git a/backend/dissemination/workbooklib/end_to_end_core.py b/backend/dissemination/workbooklib/end_to_end_core.py index f0798501cc..bb83b64149 100644 --- a/backend/dissemination/workbooklib/end_to_end_core.py +++ b/backend/dissemination/workbooklib/end_to_end_core.py @@ -8,7 +8,8 @@ import jwt import requests from pprint import pprint -from datetime import datetime +from datetime import datetime, timezone +from audit.models import SingleAuditChecklist from dissemination.workbooklib.workbook_creation import ( sections, @@ -43,11 +44,18 @@ def step_through_certifications(sac): - sac.transition_to_ready_for_certification() - sac.transition_to_auditor_certified() - sac.transition_to_auditee_certified() - sac.transition_to_submitted() - sac.transition_to_disseminated() + stati = [ + SingleAuditChecklist.STATUS.IN_PROGRESS, + SingleAuditChecklist.STATUS.READY_FOR_CERTIFICATION, + SingleAuditChecklist.STATUS.AUDITOR_CERTIFIED, + SingleAuditChecklist.STATUS.AUDITEE_CERTIFIED, + SingleAuditChecklist.STATUS.CERTIFIED, + SingleAuditChecklist.STATUS.SUBMITTED, + SingleAuditChecklist.STATUS.DISSEMINATED, + ] + for status in stati: + sac.transition_name.append(status) + sac.transition_date.append(datetime.now(timezone.utc)) sac.save() @@ -189,7 +197,7 @@ def api_check(json_test_tables): return combined_summary -def generate_workbooks(user, email, dbkey, year): +def generate_workbooks(user, email, dbkey, year, store_files=True, run_api_checks=True): entity_id = "DBKEY {dbkey} {year} {date:%Y_%m_%d_%H_%M_%S}".format( dbkey=dbkey, year=year, date=datetime.now() ) @@ -197,32 +205,28 @@ def generate_workbooks(user, email, dbkey, year): if sac.general_information["audit_type"] == "alternative-compliance-engagement": print(f"Skipping ACE audit: {dbkey}") else: - loader = workbook_loader(user, sac, dbkey, year, entity_id) + _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf", store_files) + + loader = workbook_loader(user, sac, dbkey, year, entity_id, store_files) json_test_tables = [] for section, fun in sections.items(): - # FIXME: Can we conditionally upload the addl' and secondary workbooks? (_, json, _) = loader(fun, section) json_test_tables.append(json) - _post_upload_pdf(sac, user, "audit/fixtures/basic.pdf") - step_through_certifications(sac) - - # shaped_sac = sac_validation_shape(sac) - # result = submission_progress_check(shaped_sac, sar=None, crossval=False) - # print(result) + step_through_certifications(sac) errors = sac.validate_cross() pprint(errors.get("errors", "No errors found in cross validation")) disseminate(sac, year) - # pprint(json_test_tables) - combined_summary = api_check(json_test_tables) - logger.info(combined_summary) + if run_api_checks: + combined_summary = api_check(json_test_tables) + logger.info(combined_summary) -def run_end_to_end(email, dbkey, year): +def run_end_to_end(email, dbkey, year, store_files=True, run_api_checks=True): try: user = User.objects.get(email=email) except User.DoesNotExist: logger.info("No user found for %s, have you logged in once?", email) return - generate_workbooks(user, email, dbkey, year) + generate_workbooks(user, email, dbkey, year, store_files, run_api_checks) diff --git a/backend/dissemination/workbooklib/sac_creation.py b/backend/dissemination/workbooklib/sac_creation.py index 7d2217c5e4..cf2e0f60e0 100644 --- a/backend/dissemination/workbooklib/sac_creation.py +++ b/backend/dissemination/workbooklib/sac_creation.py @@ -292,7 +292,7 @@ def _make_excel_file(filename, f_obj): return file -def _post_upload_pdf(this_sac, this_user, pdf_filename): +def _post_upload_pdf(this_sac, this_user, pdf_filename, store_files=True): """Upload a workbook for this SAC. This should be idempotent if it is called on a SAC that already @@ -305,12 +305,12 @@ def _post_upload_pdf(this_sac, this_user, pdf_filename): # nothing to do here return - with open(pdf_filename, "rb") as f: - content = f.read() - file = SimpleUploadedFile(pdf_filename, content, "application/pdf") - print(file.__dict__) + # with open(pdf_filename, "rb") as f: + # content = f.read() + # file = SimpleUploadedFile(pdf_filename, content, "application/pdf") + # print(file.__dict__) pdf_file = PDFFile( - file=file, + file=SimpleUploadedFile("pdf.pdf", b"", "application/pdf"), component_page_numbers={ "financial_statements": 1, "financial_statements_opinion": 2, @@ -327,15 +327,18 @@ def _post_upload_pdf(this_sac, this_user, pdf_filename): sac_id=this_sac.id, ) - validator_mapping["PDF"](pdf_file.file) - - pdf_file.full_clean() - pdf_file.save() + # Sometimes, we just want to generate data for testing, + # not store the artifacts. + # FIXME: We would have to... the save stores data in a table, + # but it also creates a file. I'm going to eat having 0-length PDFs for now. + if True: + # pdf_file.full_clean() + pdf_file.save() this_sac.save() -def _post_upload_workbook(this_sac, this_user, section, xlsx_file): +def _post_upload_workbook(this_sac, this_user, section, xlsx_file, store_files=True): """Upload a workbook for this SAC. This should be idempotent if it is called on a SAC that already @@ -358,12 +361,24 @@ def _post_upload_workbook(this_sac, this_user, section, xlsx_file): sac_id=this_sac.id, form_section=section, ) - excel_file.full_clean() - excel_file.save() + + # Sometimes, we just want to generate data for testing, + # not store the artifacts. + if store_files: + excel_file.full_clean() + excel_file.save() audit_data = extract_mapping[section](excel_file.file) validator_mapping[section](audit_data) + this_sac = _post_upload_workbook_assign_data(section, this_sac, audit_data) + + this_sac.save() + + logger.info(f"Created {section} workbook upload for SAC {this_sac.id}") + + +def _post_upload_workbook_assign_data(section, this_sac, audit_data): if section == FORM_SECTIONS.FEDERAL_AWARDS_EXPENDED: this_sac.federal_awards = audit_data elif section == FORM_SECTIONS.FINDINGS_UNIFORM_GUIDANCE: @@ -380,7 +395,4 @@ def _post_upload_workbook(this_sac, this_user, section, xlsx_file): this_sac.additional_ueis = audit_data elif section == FORM_SECTIONS.ADDITIONAL_EINS: this_sac.additional_eins = audit_data - - this_sac.save() - - logger.info(f"Created {section} workbook upload for SAC {this_sac.id}") + return this_sac diff --git a/backend/dissemination/workbooklib/workbook_creation.py b/backend/dissemination/workbooklib/workbook_creation.py index 580eacfa3e..25198dd21b 100644 --- a/backend/dissemination/workbooklib/workbook_creation.py +++ b/backend/dissemination/workbooklib/workbook_creation.py @@ -76,7 +76,7 @@ def setup_sac(user, test_name, dbkey): return sac -def workbook_loader(user, sac, dbkey, year, entity_id): +def workbook_loader(user, sac, dbkey, year, entity_id, store_files=True): def _loader(workbook_generator, section): with MemoryFS() as mem_fs: filename = filenames[section].format(dbkey) @@ -86,7 +86,7 @@ def _loader(workbook_generator, section): outfile = mem_fs.openbin(filename, mode="r") excel_file = _make_excel_file(filename, outfile) if user: - _post_upload_workbook(sac, user, section, excel_file) + _post_upload_workbook(sac, user, section, excel_file, store_files) outfile.close() return (wb, json, filename) diff --git a/backend/report_submission/templates/report_submission/step-2.html b/backend/report_submission/templates/report_submission/step-2.html index 9f91fdba89..4227f5a1bd 100644 --- a/backend/report_submission/templates/report_submission/step-2.html +++ b/backend/report_submission/templates/report_submission/step-2.html @@ -160,7 +160,11 @@

aria-describedby="uei-search-result-description">
- loading + + +

diff --git a/backend/static/js/search-results-pagination.js b/backend/static/js/search-results-pagination.js deleted file mode 100644 index e167d33db3..0000000000 --- a/backend/static/js/search-results-pagination.js +++ /dev/null @@ -1,68 +0,0 @@ -var FORM = document.getElementById('search-form'); -const pagination_links = document.querySelectorAll('[aria-label^="Page"]'); -const next_page_link = document.querySelectorAll('[aria-label="Next page"]'); -const previous_page_link = document.querySelectorAll( - '[aria-label="Previous page"]' -); - -function attachEventHandlers() { - // If any pagination links are clicked, set the page form element and submit it for a reload - pagination_links.forEach((link) => { - link.addEventListener('click', (e) => { - e.preventDefault(); - FORM.elements['page'].value = link.textContent; - FORM.submit(); - }); - }); - - // If the next or previous page buttons are clicked, set the page form element to be +/- 1 and submit for a reload - if (next_page_link[0]) { - next_page_link[0].addEventListener('click', (e) => { - e.preventDefault(); - FORM.elements['page'].value = parseInt(FORM.elements['page'].value) + 1; - FORM.submit(); - }); - } - if (previous_page_link[0]) { - previous_page_link[0].addEventListener('click', (e) => { - e.preventDefault(); - FORM.elements['page'].value = parseInt(FORM.elements['page'].value) - 1; - FORM.submit(); - }); - } - - // The form resets to the default value, which is whatever the user entered on the last form submission, _not_ fully empty fields. - // We cannot just loop over every form element, because buttons and hidden inputs should be handled differently. - FORM.addEventListener('reset', (e) => { - e.preventDefault(); - // Empty out textareas - var text_areas = document.getElementsByTagName('textarea'); - Array.from(text_areas).forEach((input) => { - input.value = ''; - }); - // Uncheck checkboxes - var checkboxes = document.querySelectorAll('[type="checkbox"]'); - Array.from(checkboxes).forEach((checkbox) => { - checkbox.checked = false; - }); - // Wipe FAC release dates - var start_date = document.getElementById('start-date'); - var end_date = document.getElementById('end-date'); - start_date.value = ''; - end_date.value = ''; - // Reset Cog/Over dropdown - var default_option = document.getElementById('cog_or_oversight--none'); - default_option.selected = true; - // Wipe agency name - var agency_name = document.getElementById('agency-name'); - agency_name.value = ''; - - FORM.reset(); - }); -} - -function init() { - attachEventHandlers(); -} - -window.onload = init; diff --git a/backend/static/js/search-results.js b/backend/static/js/search-results.js new file mode 100644 index 0000000000..bb7afee90c --- /dev/null +++ b/backend/static/js/search-results.js @@ -0,0 +1,121 @@ +var FORM = document.getElementById('search-form'); +const pagination_links = document.querySelectorAll('[aria-label^="Page"]'); +const next_page_link = document.querySelectorAll('[aria-label="Next page"]'); +const previous_page_link = document.querySelectorAll( + '[aria-label="Previous page"]' +); + +/* + If any pagination links are clicked, set the page form element and submit it for a reload. + If the next or previous page buttons are clicked, set the page form element to be +/- 1 and submit for a reload. +*/ +function attachEventHandlersPagination() { + pagination_links.forEach((link) => { + link.addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = link.textContent; + FORM.submit(); + }); + }); + + if (next_page_link[0]) { + next_page_link[0].addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = parseInt(FORM.elements['page'].value) + 1; + FORM.submit(); + }); + } + if (previous_page_link[0]) { + previous_page_link[0].addEventListener('click', (e) => { + e.preventDefault(); + FORM.elements['page'].value = parseInt(FORM.elements['page'].value) - 1; + FORM.submit(); + }); + } +} + +/* + The standard form behavior resets to the default value, which is whatever the user entered on the last form submission, _not_ fully empty fields. + We cannot just loop over every form element, because buttons and hidden inputs should be handled differently. +*/ +function attachEventHandlersReset() { + FORM.addEventListener('reset', (e) => { + e.preventDefault(); + // Empty out textareas + var text_areas = document.getElementsByTagName('textarea'); + Array.from(text_areas).forEach((input) => { + input.value = ''; + }); + // Uncheck checkboxes + var checkboxes = document.querySelectorAll('[type="checkbox"]'); + Array.from(checkboxes).forEach((checkbox) => { + checkbox.checked = false; + }); + // Wipe FAC release dates + var start_date = document.getElementById('start-date'); + var end_date = document.getElementById('end-date'); + start_date.value = ''; + end_date.value = ''; + // Reset Cog/Over dropdown + var default_cog_over_option = document.getElementById( + 'cog_or_oversight--none' + ); + default_cog_over_option.selected = true; + // Wipe agency name + var agency_name = document.getElementById('agency-name'); + agency_name.value = ''; + // Reset state dropdown + var default_state_option = document.getElementById('state--none'); + default_state_option.selected = true; + + FORM.reset(); + }); +} + +/* + Get the table headers and their sort buttons. + Using the ID of the table headers, attach event handlers to re-submit the form when clicking them. + This will reload the page with the associated sort values, so that searches appear to sort across many pages. + + If the sort buttons get clicked, we have one of three states: + 1. We are not sorting, and we move to sort in descending. + 2. We are sorting by descending, and we move to sort in ascending. + 3. We are sorting by ascending, and we move to no sort at all. + i. In this case, we wipe both the order_by and order_direction fields. Or, it will maintain the values and sort anyway. + In any case, we want to reset the page number to one. +*/ +function attachEventHandlersSorting() { + var FORM = document.getElementById('search-form'); + var table_headers = document.querySelectorAll('th[id]'); + + table_headers.forEach((header) => { + var button = header.querySelector('button'); + var current_sort = header.getAttribute('aria-sort'); + + button.addEventListener('click', (e) => { + e.preventDefault(); + + if (current_sort == 'ascending') { + FORM.elements['order_direction'].value = ''; + FORM.elements['order_by'].value = ''; + } else if (current_sort == 'descending') { + FORM.elements['order_by'].value = header.id; + FORM.elements['order_direction'].value = 'ascending'; + } else { + FORM.elements['order_by'].value = header.id; + FORM.elements['order_direction'].value = 'descending'; + } + FORM.elements['page'].value = 1; + + FORM.submit(); + }); + }); +} + +function init() { + attachEventHandlersPagination(); + attachEventHandlersReset(); + attachEventHandlersSorting(); +} + +window.onload = init;

NameUEI or EINAcc DateAYCog or OverView PDFFinding my ALNFinding all ALN