From 80f1db8e18e13d3e0e66a119acc56696781db672 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 12 Jan 2024 20:39:28 -0500 Subject: [PATCH 1/5] fix bill_text template. This had some out of date references which was causing the full text search to miss bills like in #388. closes #388 --- .../commands/preview_search_text.py | 30 +++++++++++++++++++ chicago/search_indexes.py | 12 ++------ .../councilmatic_core => }/bill_text.txt | 13 ++++---- chicago/templatetags/extras.py | 9 ------ 4 files changed, 41 insertions(+), 23 deletions(-) create mode 100644 chicago/management/commands/preview_search_text.py rename chicago/templates/search/{indexes/councilmatic_core => }/bill_text.txt (57%) diff --git a/chicago/management/commands/preview_search_text.py b/chicago/management/commands/preview_search_text.py new file mode 100644 index 0000000..9e3ecfd --- /dev/null +++ b/chicago/management/commands/preview_search_text.py @@ -0,0 +1,30 @@ +from django.core.management.base import BaseCommand +from chicago.models import ChicagoBill +from chicago.search_indexes import BillIndex +import pprint + + +class Command(BaseCommand): + """A simple management command which clears the site-wide cache.""" + + help = "Preview the text template that will be used in the search." + + def add_arguments(self, parser): + parser.add_argument( + "identifier", + help=( + "The identifier of the bill to preview" + ), + ) + + + + def handle(self, *args, **kwargs): + + + + + b = ChicagoBill.objects.get(identifier=kwargs['identifier']) + + self.stdout.write(pprint.pformat(BillIndex().prepare(b), indent=4)) + diff --git a/chicago/search_indexes.py b/chicago/search_indexes.py index de73d71..1adf026 100644 --- a/chicago/search_indexes.py +++ b/chicago/search_indexes.py @@ -15,18 +15,15 @@ class BillIndex(indexes.SearchIndex, indexes.Indexable): text = indexes.CharField( document=True, use_template=True, - template_name="search/indexes/councilmatic_core/bill_text.txt", + template_name="search/bill_text.txt", ) slug = indexes.CharField(model_attr="slug", indexed=False) id = indexes.CharField(model_attr="id", indexed=False) bill_type = indexes.CharField(faceted=True) identifier = indexes.CharField(model_attr="identifier") - description = indexes.CharField(model_attr="title", boost=1.25) + title = indexes.CharField(model_attr="title", boost=1.25) source_url = indexes.CharField(model_attr="sources__url", indexed=False) source_note = indexes.CharField(model_attr="sources__note") - abstract = indexes.CharField( - model_attr="abstracts__abstract", boost=1.25, default="" - ) friendly_name = indexes.CharField() sort_name = indexes.CharField(faceted=True) @@ -38,7 +35,7 @@ class BillIndex(indexes.SearchIndex, indexes.Indexable): last_action_date = indexes.DateTimeField() inferred_status = indexes.CharField(faceted=True) legislative_session = indexes.CharField(faceted=True) - topics = indexes.MultiValueField(faceted=True) + topics = indexes.MultiValueField(model_attr="topics", faceted=True) def get_model(self): return ChicagoBill @@ -60,9 +57,6 @@ def prepare(self, obj): return data - def prepare_topics(self, obj): - return obj.topics - def prepare_last_action_date(self, obj): if not obj.last_action_date: action_dates = [a.date for a in obj.actions.all()] diff --git a/chicago/templates/search/indexes/councilmatic_core/bill_text.txt b/chicago/templates/search/bill_text.txt similarity index 57% rename from chicago/templates/search/indexes/councilmatic_core/bill_text.txt rename to chicago/templates/search/bill_text.txt index 94d93e7..00961bb 100644 --- a/chicago/templates/search/indexes/councilmatic_core/bill_text.txt +++ b/chicago/templates/search/bill_text.txt @@ -1,9 +1,12 @@ {% load extras %} -{{ object.identifier | alternative_identifiers }} {{ object.friendly_name }} -{{ object.classification }} -{{ object.description }} -{{ object.abstract }} +{% for identifier in object.other_identifiers.all %} + {{ identifier.identifier }} +{% endfor %} +{% for classification in object.classification %} +{{ classification }} +{% endfor %} +{{ object.title }} {% for s in object.sponsorships.all %} {% if s.primary %} {{ s.person.name }} @@ -16,4 +19,4 @@ {% for t in object.topics %} {{t}} {% endfor %} -{{ object.ocr_full_text|clean_html }} +{{ object.ocr_full_text }} diff --git a/chicago/templatetags/extras.py b/chicago/templatetags/extras.py index 15003eb..c006804 100644 --- a/chicago/templatetags/extras.py +++ b/chicago/templatetags/extras.py @@ -96,15 +96,6 @@ def clean_html(text): return re.sub(r"&(?:\w+|#\d+);", "", value) -@register.filter -@stringfilter -def alternative_identifiers(id_original): - id_1 = re.sub(" ", " 0", id_original) - id_2 = re.sub(" ", "", id_original) - id_3 = re.sub(" ", "", id_1) - return " ".join(i for i in set([id_original, id_1, id_2, id_3])) - - @register.filter def get_item(dictionary, key): return dictionary.get(key) From a5fb67fb1b383da69f44ddd2c0fcaa56dea2b478 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 12 Jan 2024 20:45:03 -0500 Subject: [PATCH 2/5] blackify --- chicago/management/commands/preview_search_text.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/chicago/management/commands/preview_search_text.py b/chicago/management/commands/preview_search_text.py index 9e3ecfd..5bf495d 100644 --- a/chicago/management/commands/preview_search_text.py +++ b/chicago/management/commands/preview_search_text.py @@ -12,19 +12,11 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( "identifier", - help=( - "The identifier of the bill to preview" - ), + help=("The identifier of the bill to preview"), ) - - def handle(self, *args, **kwargs): - - - - b = ChicagoBill.objects.get(identifier=kwargs['identifier']) + b = ChicagoBill.objects.get(identifier=kwargs["identifier"]) self.stdout.write(pprint.pformat(BillIndex().prepare(b), indent=4)) - From b977c4fad0a3c1e33669e7d534792252b119d84d Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 12 Jan 2024 20:47:21 -0500 Subject: [PATCH 3/5] remove unused arg --- chicago/management/commands/preview_search_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chicago/management/commands/preview_search_text.py b/chicago/management/commands/preview_search_text.py index 5bf495d..4b8149c 100644 --- a/chicago/management/commands/preview_search_text.py +++ b/chicago/management/commands/preview_search_text.py @@ -19,4 +19,4 @@ def handle(self, *args, **kwargs): b = ChicagoBill.objects.get(identifier=kwargs["identifier"]) - self.stdout.write(pprint.pformat(BillIndex().prepare(b), indent=4)) + self.stdout.write(pprint.pformat(BillIndex().prepare(b)) From 023f7b40eccea0e4710b15f845753537b3b99369 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 12 Jan 2024 20:48:02 -0500 Subject: [PATCH 4/5] fix imoprt order --- chicago/management/commands/preview_search_text.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/chicago/management/commands/preview_search_text.py b/chicago/management/commands/preview_search_text.py index 4b8149c..cbff81e 100644 --- a/chicago/management/commands/preview_search_text.py +++ b/chicago/management/commands/preview_search_text.py @@ -1,7 +1,9 @@ +import pprint + from django.core.management.base import BaseCommand + from chicago.models import ChicagoBill from chicago.search_indexes import BillIndex -import pprint class Command(BaseCommand): From f5ba1b6eb6d00353773e586ce961ccca6fd87cb4 Mon Sep 17 00:00:00 2001 From: Forest Gregg Date: Fri, 12 Jan 2024 21:39:00 -0500 Subject: [PATCH 5/5] update title --- chicago/templates/feeds/search_item_description.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chicago/templates/feeds/search_item_description.html b/chicago/templates/feeds/search_item_description.html index b50fa3e..45b49c0 100644 --- a/chicago/templates/feeds/search_item_description.html +++ b/chicago/templates/feeds/search_item_description.html @@ -14,7 +14,7 @@ - {{ obj.description }} + {{ obj.title }}

Sponsors: