Search, Advanced Search, Oh My! (#3552)

* Enabling advanced search

This leaves a basic search in place, and adds an advanced search.

Per #3526 for more information and
documentation at the top of search.html:

All input fields besides UEI/EIN are commented out pending a performance investigation and fix.
Summary report downloads are also disabled.

** SUMMARY REPORTS HAVE BEEN RE-ENABLED**

However, this commit missed the SF-SAC on overview pages. TBD.

Alerts have been put in place.

** ALERTS HAVE BEEN COMMENTED OUT **

To undo this action, complete the following steps.
Search filters:
    1. Uncomment the search filters

** SEARCH FILTERS UNCOMMENTED FROM search/advanced **

Warnings:
    1. Remove search-alert-performance.html and its imports.

** THIS WAS COMMENTED OUT **

    2. Remove the warning banner in header.html

** THIS WAS ALSO COMMENTED OUT **

Summary report downloads:
    1. Uncomment the summary report download button here

** THIS WAS UNUNCOMMENTED **

    2. Uncomment the SF-SAC download button in summary.html

** THIS WAS NOT DONE - NEXT COMMIT **

    3. Remove the forced Http404 in SingleSummaryReportDownloadView (dissemination/views.py)

** I HAVE NO IDEA IF THIS WAS DONE... TBD **

* Uncommenting the SF-SAC on summary pages

* Correctly flip between basic/advanced

Adds a flag for advanced search.

* Linting results.

* Fix linting

* A fix on the adv search code

And test.

* Updating text

* Update search-alert-info.html

Copy edits to "Basic Search" and "Advanced Search" sentences.

* Enabling single summary report download

* cleanup

* update views for advanced search

* lint

* general params and MP logic

* invert the general params

* invert the inversion

* update test_search

* fix summary download tests in test_views

---------

Co-authored-by: Hassan D. M. Sambo <[email protected]>
Co-authored-by: Laura H <[email protected]>
Co-authored-by: Daniel Swick <[email protected]>

backend/dissemination/forms.py

@@ -1,7 +1,7 @@
 from django import forms
 
 
-class SearchForm(forms.Form):
+class AdvancedSearchForm(forms.Form):
     # Multiple choice field mappings
     findings_field_mapping = {
         "field_name": [
@@ -134,3 +134,76 @@ def clean_limit(self):
         Default page limit to 30.
         """
         return int(self.cleaned_data["limit"] or 30)
+
+
+class SearchForm(forms.Form):
+    # Multiple choice field Tuples. "choices" variable in field declaration.
+    AY_choices = (("all_years", "All years"),) + tuple(
+        (x, str(x)) for x in reversed(range(2016, 2024))
+    )
+
+    # Query params
+    entity_name = forms.CharField(required=False)
+    uei_or_ein = forms.CharField(required=False)
+    start_date = forms.DateField(required=False)
+    end_date = forms.DateField(required=False)
+    agency_name = forms.CharField(required=False)
+    audit_year = forms.MultipleChoiceField(
+        choices=AY_choices, initial=[2023], required=False
+    )
+    auditee_state = forms.CharField(required=False)
+
+    # Display params
+    limit = forms.CharField(required=False)
+    page = forms.CharField(required=False)
+    order_by = forms.CharField(required=False)
+    order_direction = forms.CharField(required=False)
+
+    # Variables for cleaning functions
+    text_input_delimiters = [
+        ",",
+        ":",
+        ";",
+        "-",
+        " ",
+    ]
+
+    def clean_entity_name(self):
+        """
+        Clean the name field. We can't trust that separators aren't a part of a name somewhere,
+        so just split on newlines.
+        """
+        text_input = self.cleaned_data["entity_name"]
+        return text_input.splitlines()
+
+    def clean_uei_or_ein(self):
+        """
+        Clean up the UEI/EIN field. Replace common separators with a newline.
+        Split on the newlines. Strip all the resulting elements.
+        """
+        text_input = self.cleaned_data["uei_or_ein"]
+        for delimiter in self.text_input_delimiters:
+            text_input = text_input.replace(delimiter, "\n")
+        text_input = [x.strip() for x in text_input.splitlines()]
+        return text_input
+
+    def clean_audit_year(self):
+        """
+        If "All years" is selected, don't include any years.
+        """
+        audit_year = self.cleaned_data["audit_year"]
+        if "all_years" in audit_year:
+            return []
+        return audit_year
+
+    def clean_page(self):
+        """
+        Default page number to one.
+        """
+        return int(self.cleaned_data["page"] or 1)
+
+    def clean_limit(self):
+        """
+        Default page limit to 30.
+        """
+        return int(self.cleaned_data["limit"] or 30)

backend/dissemination/search.py

@@ -6,7 +6,7 @@
 from .searchlib.search_findings import search_findings
 from .searchlib.search_direct_funding import search_direct_funding
 from .searchlib.search_major_program import search_major_program
-from dissemination.models import DisseminationCombined
+from dissemination.models import DisseminationCombined, General
 
 logger = logging.getLogger(__name__)
 
@@ -16,22 +16,26 @@
 # Their ORM cookbook looks to be useful reading.
 # https://books.agiliq.com/projects/django-orm-cookbook/en/latest/subquery.html
 
+# {'alns': [], -- DisseminationCombined
+#  'names': ['AWESOME'], -- General
+#  'uei_or_eins': [],  -- General
+#  'start_date': None,  -- General
+#  'end_date': None,  -- General
+#  'cog_or_oversight': '', -- General, but not wanted
+#  'agency_name': '',  -- NO IDEA
+#  'audit_years': [], -- General
+#  'findings': [], -- DisseminationCombined
+#  'direct_funding': [], -- DisseminationCombined
+#  'major_program': [], -- DisseminationCombined
+#  'auditee_state': '', -- General
+#  'order_by': -- General
+#  'fac_accepted_date', -- General
+#  'order_direction': -- General
+#  'descending', 'LIMIT': 1000} -- General
 
-def is_only_general_params(params_dict):
-    params_set = set(list(params_dict.keys()))
-    gen_set = set(
-        [
-            "audit_years",
-            "auditee_state",
-            "names",
-            "uei_or_eins",
-            "start_date",
-            "end_date",
-            "agency_name",
-            "cog_or_oversight",
-        ]
-    )
-    return params_set.issubset(gen_set)
+
+def is_advanced_search(params_dict):
+    return params_dict.get("advanced_search_flag")
 
 
 def search(params):
@@ -49,16 +53,19 @@ def search(params):
     ##############
     # GENERAL
 
-    if is_only_general_params(params):
-        results = search_general(DisseminationCombined, params)
-        results = _sort_results(results, params)
-    else:
+    logger.info(params)
+    if is_advanced_search(params):
+        logger.info("search Searching `DisseminationCombined`")
         results = search_general(DisseminationCombined, params)
         results = _sort_results(results, params)
         results = search_alns(results, params)
         results = search_findings(results, params)
         results = search_direct_funding(results, params)
         results = search_major_program(results, params)
+    else:
+        logger.info("search Searching `General`")
+        results = search_general(General, params)
+        results = _sort_results(results, params)
 
     results = results.distinct("report_id", params.get("order_by", "fac_accepted_date"))
 

backend/dissemination/searchlib/search_major_program.py

@@ -15,9 +15,9 @@ def search_major_program(general_results, params):
     q = Q()
     major_program_fields = params.get("major_program", [])
 
-    if True in major_program_fields:
+    if "True" in major_program_fields:
         q |= Q(is_major="Y")
-    elif False in major_program_fields:
+    elif "False" in major_program_fields:
         q |= Q(is_major="N")
 
     filtered_general_results = general_results.filter(q).distinct()