Search Results - Summary Report Generation (#3025)

* Multiple report generation, first pass * Comments, formatting, minor cleanup * Summary reports query by model, not report_id * XLSX, not CSV * Cannot edit a namedtuple after initializing it. * FAC API Link in coversheet * Alert copy, download button only on < 1000 results * Use padding instead of an empty tag. * Update note copy. * Add download limit to config.settings, use it everywhere * Small summary report protection * 404 and 400 errors for summary report downloads. * Tests for summary report downloads. * Copy - Alert heading & remove API links. * Linting - Imports, f-strings, logger values. * All infobox info at the same time, all the time. Broken into a component.
GSA-TTS · Dec 22, 2023 · b1e0a77 · b1e0a77
1 parent e0a034d
commit b1e0a77
Show file tree

Hide file tree

Showing 8 changed files with 383 additions and 110 deletions.
diff --git a/backend/config/settings.py b/backend/config/settings.py
@@ -531,8 +531,10 @@
 OMB_EXP_DATE = "09/30/2026"
 
 # APP-level constants
-DOLLAR_THRESHOLD = 750000
 CENSUS_DATA_SOURCE = "CENSUS"
+GSA_MIGRATION = "GSA_MIGRATION"  # There is a copy of `GSA_MIGRATION` in Base.libsonnet. If you change it here, change it there too.
+DOLLAR_THRESHOLD = 750000
+SUMMARY_REPORT_DOWNLOAD_LIMIT = 1000
 
 # A version of these regexes also exists in Base.libsonnet
 REGEX_ALN_PREFIX = r"^([0-9]{2})$"

diff --git a/backend/dissemination/search.py b/backend/dissemination/search.py
@@ -36,6 +36,10 @@ def search_general(
     order_by=ORDER_BY.fac_accepted_date,
     order_direction=DIRECTION.ascending,
 ):
+    """
+    Given any (or no) search fields, build and execute a query on the General table and return the results.
+    Empty searches return everything.
+    """
     if not order_by:
         order_by = ORDER_BY.fac_accepted_date
     if not order_direction:

diff --git a/backend/dissemination/summary_reports.py b/backend/dissemination/summary_reports.py
@@ -281,6 +281,15 @@ def insert_precert_coversheet(workbook):
 def insert_dissem_coversheet(workbook):
     sheet = workbook.create_sheet("Coversheet", 0)
     sheet.append(["Time created", datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")])
+    sheet.append(
+        [
+            "Note",
+            f"This spreadsheet contains the first {settings.SUMMARY_REPORT_DOWNLOAD_LIMIT} results of your search. If you need to download more than {settings.SUMMARY_REPORT_DOWNLOAD_LIMIT} submissions, try limiting your search parameters to download in batches.",
+        ]
+    )
+    # Uncomment if we want to link to the FAC API for larger data dumps.
+    # sheet.cell(row=3, column=2).value = "FAC API Link"
+    # sheet.cell(row=3, column=2).hyperlink = f"{settings.STATIC_SITE_URL}/developers/"
     set_column_widths(sheet)
 
 
@@ -317,12 +326,11 @@ def gather_report_data_dissemination(report_ids):
 
         data[model_name] = {"field_names": field_names, "entries": []}
 
-        for report_id in report_ids:
-            objects = model.objects.all().filter(report_id=report_id)
-            for obj in objects:
-                data[model_name]["entries"].append(
-                    [getattr(obj, field_name) for field_name in field_names]
-                )
+        objects = model.objects.all().filter(report_id__in=report_ids)
+        for obj in objects:
+            data[model_name]["entries"].append(
+                [getattr(obj, field_name) for field_name in field_names]
+            )
     return data
 
 

diff --git a/backend/dissemination/templates/search-alert-info.html b/backend/dissemination/templates/search-alert-info.html
@@ -0,0 +1,21 @@
+{% load humanize %}
+{% load sprite_helper %}
+<div class="usa-alert usa-alert--info">
+    <div class="usa-alert__body">
+        <h4 class="usa-alert__heading">Searching the FAC database</h4>
+        <p class="usa-alert__text padding-bottom-2">
+            Learn more about how our search filters work on <a href='https://www.fac.gov/data/resources/' target='_blank'>our Search Resources page</a>.
+        </p>
+
+        <h4 class="usa-alert__heading">Sorting</h4>
+        <p class="usa-alert__text padding-bottom-2">
+            Use the arrows at the top of each column to sort results.
+        </p>
+
+        <h4 class="usa-alert__heading">Summary reports</h4>
+        <p class="usa-alert__text">
+            For search results of {{ summary_report_download_limit|intcomma }} submissions or less, you can download a combined spreadsheet of all data. 
+            If you need to download more than {{ summary_report_download_limit|intcomma }} submissions, try limiting your search parameters to download in batches.
+        </p>
+    </div>
+</div>
diff --git a/backend/dissemination/templates/search.html b/backend/dissemination/templates/search.html
@@ -170,27 +170,36 @@ <h3>Filters</h3>
             <div class="grid-col audit-search-results">
                 <h2 class="font-sans-2xl">Search single audit reports</h2>
                 {% if results|length > 0 %}
-                    <div class="usa-alert usa-alert--info">
-                        <div class="usa-alert__body">
-                            <h4 class="usa-alert__heading">Sorting</h4>
-                            <p class="usa-alert__text">
-                                Use the arrows at the top of each column to sort results.
-                            </p>
-                        </div>
+                    {% include "search-alert-info.html"%}
+                    <div class="margin-y-2 grid-row display-flex flex-justify">
+                        <p class="margin-0 flex-align-self-center">
+                            <strong>Results: {{ results_count }}</strong>
+                            <span class="margin-left-2 text-normal text-italic">showing {{ limit }} per page</span>
+                        </p>
+                        {% if results_count <= summary_report_download_limit %}
+                            <button class="usa-button display-flex"
+                                    formaction="{% url 'dissemination:MultipleSummaryReportDownload' %}"
+                                    form="search-form"
+                                    value="Search">
+                                <svg class="usa-icon margin-right-1 flex-align-self-center"
+                                    aria-hidden="true"
+                                    role="img">
+                                    {% uswds_sprite "file_download" %}
+                                </svg>
+                                <p class="margin-0">Download all</p>
+                            </button>
+                        {% endif %}
                     </div>
-                    <p class="margin-y-4">
-                        <strong>Results: {{ results_count }}</strong>
-                        <span class="margin-left-2 text-normal text-italic">showing {{ limit }} per page</span>
-                    </p>
+
                     <div class="usa-table-container--scrollable margin-0" tabindex="0">
                         <table class="usa-table usa-table--striped usa-table--compact width-full">
                             <thead>
                                 <tr>
-                                    {% include "search-table-header.html" with friendly_title="Name" field_name="auditee_name"%}
-                                    {% include "search-table-header.html" with friendly_title="UEI or EIN" field_name="auditee_uei"%}
-                                    {% include "search-table-header.html" with friendly_title="Acc Date" field_name="fac_accepted_date"%}
-                                    {% include "search-table-header.html" with friendly_title="AY" field_name="audit_year"%}
-                                    {% include "search-table-header.html" with friendly_title="Cog or Over" field_name="cog_over"%}
+                                    {% include "search-table-header.html" with friendly_title="Name" field_name="auditee_name" %}
+                                    {% include "search-table-header.html" with friendly_title="UEI or EIN" field_name="auditee_uei" %}
+                                    {% include "search-table-header.html" with friendly_title="Acc Date" field_name="fac_accepted_date" %}
+                                    {% include "search-table-header.html" with friendly_title="AY" field_name="audit_year" %}
+                                    {% include "search-table-header.html" with friendly_title="Cog or Over" field_name="cog_over" %}
                                     <th scope="col" role="columnheader">View</th>
                                     <th scope="col" role="columnheader">PDF</th>
                                     {% if results.0.finding_my_aln is not None%}
@@ -305,13 +314,8 @@ <h4 class="usa-alert__heading">Sorting</h4>
                         </ul>
                     </nav>
                 {% elif results is not None %}
+                    {% include "search-alert-info.html"%}
                     <div class="usa-alert usa-alert--info">
-                        <div class="usa-alert__body">
-                            <h4 class="usa-alert__heading">Searching the FAC database</h4>
-                            <p class="usa-alert__text">
-                                Learn more about how our search filters work on <a href='https://www.fac.gov/data/resources/' target='_blank'>our Search Resources page</a>.
-                            </p>
-                        </div>
                     </div>
                     <div class="search-instructions">
                         <img src="{% static 'img/circle-arrow.svg' %}"
@@ -321,14 +325,7 @@ <h4 class="usa-alert__heading">Searching the FAC database</h4>
                         </p>
                     </div>
                 {% else %}
-                    <div class="usa-alert usa-alert--info">
-                        <div class="usa-alert__body">
-                            <h4 class="usa-alert__heading">Searching the FAC database</h4>
-                            <p class="usa-alert__text">
-                                Learn more about how our search filters work on <a href='https://www.fac.gov/data/resources/' target='_blank'>our Search Resources page</a>.
-                            </p>
-                        </div>
-                    </div>
+                    {% include "search-alert-info.html"%}
                     <div class="search-instructions">
                         <img src="{% static 'img/circle-arrow.svg' %}"
                             alt="an arrow points left, toward the search form" />

diff --git a/backend/dissemination/test_views.py b/backend/dissemination/test_views.py
@@ -448,3 +448,140 @@ def test_summary_context(self):
             response.context["data"]["Notes to SEFA"][0]["accounting_policies"],
             note.accounting_policies,
         )
+
+
+class SummaryReportDownloadViewTests(TestCase):
+    def setUp(self):
+        self.anon_client = Client()
+        self.perm_client = Client()
+
+        self.perm_user = baker.make(User)
+        permission = Permission.objects.get(slug=Permission.PermissionType.READ_TRIBAL)
+        baker.make(
+            UserPermission,
+            email=self.perm_user.email,
+            user=self.perm_user,
+            permission=permission,
+        )
+        self.perm_client.force_login(self.perm_user)
+
+    def _make_general(self, is_public=True, **kwargs):
+        """
+        Create a General object in dissemination with the keyword arguments passed in.
+        """
+        general = baker.make(
+            General,
+            is_public=is_public,
+            **kwargs,
+        )
+        return general
+
+    def _summary_report_url(self):
+        return reverse("dissemination:MultipleSummaryReportDownload")
+
+    def _mock_filename(self):
+        return "some-report-name.xlsx"
+
+    def _mock_download_url(self):
+        return "http://example.com/gsa-fac-private-s3/temp/some-report-name.xlsx"
+
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_bad_search_returns_400(self, mock_persist_workbook):
+        """
+        Submitting a form with bad parameters should throw a BadRequest.
+        """
+        response = self.anon_client.post(
+            self._summary_report_url(), {"start_date": "Not a date"}
+        )
+        self.assertEquals(response.status_code, 400)
+
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_empty_results_returns_404(self, mock_persist_workbook):
+        """
+        Searches with no results should return a 404, not an empty excel file.
+        """
+        self._make_general(is_public=False, auditee_uei="123456789012")
+        response = self.anon_client.post(
+            self._summary_report_url(), {"uei_or_ein": "NotTheOther1"}
+        )
+        self.assertEquals(response.status_code, 404)
+
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_no_permissions_returns_404_on_private(self, mock_persist_workbook):
+        """
+        Non-permissioned users cannot access private audits through the summary report post.
+        """
+        self._make_general(is_public=False)
+        response = self.anon_client.post(self._summary_report_url(), {})
+        self.assertEquals(response.status_code, 404)
+
+    @patch("dissemination.views.get_download_url")
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_permissions_returns_file_on_private(
+        self, mock_persist_workbook, mock_get_download_url
+    ):
+        """
+        Permissioned users recieve a file if there are private results.
+        """
+        mock_persist_workbook.return_value = self._mock_filename()
+        mock_get_download_url.return_value = self._mock_download_url()
+
+        self._make_general(is_public=False)
+
+        response = self.perm_client.post(self._summary_report_url(), {})
+        self.assertRedirects(
+            response,
+            self._mock_download_url(),
+            status_code=302,
+            target_status_code=200,
+            fetch_redirect_response=False,
+        )
+
+    @patch("dissemination.views.get_download_url")
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_empty_search_params_returns_file(
+        self, mock_persist_workbook, mock_get_download_url
+    ):
+        """
+        File should be generated on empty search parameters ("search all").
+        """
+        mock_persist_workbook.return_value = self._mock_filename()
+        mock_get_download_url.return_value = self._mock_download_url()
+
+        self._make_general(is_public=True)
+
+        response = self.anon_client.post(self._summary_report_url(), {})
+        self.assertRedirects(
+            response,
+            self._mock_download_url(),
+            status_code=302,
+            target_status_code=200,
+            fetch_redirect_response=False,
+        )
+
+    @patch("dissemination.views.get_download_url")
+    @patch("dissemination.summary_reports.persist_workbook")
+    def test_many_results_returns_file(
+        self, mock_persist_workbook, mock_get_download_url
+    ):
+        """
+        File should still be generated if there are above SUMMARY_REPORT_DOWNLOAD_LIMIT total results.
+        """
+        mock_persist_workbook.return_value = self._mock_filename()
+        mock_get_download_url.return_value = self._mock_download_url()
+
+        for i in range(4):
+            self._make_general(
+                is_public=True,
+                report_id=generate_sac_report_id(end_date="2023-12-31", count=str(i)),
+            )
+
+        with self.settings(SUMMARY_REPORT_DOWNLOAD_LIMIT=2):
+            response = self.anon_client.post(self._summary_report_url(), {})
+            self.assertRedirects(
+                response,
+                self._mock_download_url(),
+                status_code=302,
+                target_status_code=200,
+                fetch_redirect_response=False,
+            )
diff --git a/backend/dissemination/urls.py b/backend/dissemination/urls.py
@@ -17,9 +17,14 @@
     ),
     path(
         "summary-report/xlsx/<str:report_id>",
-        views.SummaryReportDownloadView.as_view(),
+        views.SingleSummaryReportDownloadView.as_view(),
         name="SummaryReportDownload",
     ),
+    path(
+        "summary-report/xlsx",
+        views.MultipleSummaryReportDownloadView.as_view(),
+        name="MultipleSummaryReportDownload",
+    ),
     path("search/", views.Search.as_view(), name="Search"),
     path("summary/<str:report_id>", views.AuditSummaryView.as_view(), name="Summary"),
 ]