From 0a43ddf917df3d18b5290be38c13dbc47081a66a Mon Sep 17 00:00:00 2001 From: Derek Eder Date: Fri, 22 Nov 2024 16:04:02 -0600 Subject: [PATCH 1/3] adds SearchBill fulltext index, started on management command --- .../management/commands/populate_fulltext.py | 11 ++++++++++ chicago/migrations/0006_searchbill.py | 22 +++++++++++++++++++ chicago/models.py | 6 +++++ 3 files changed, 39 insertions(+) create mode 100644 chicago/management/commands/populate_fulltext.py create mode 100644 chicago/migrations/0006_searchbill.py diff --git a/chicago/management/commands/populate_fulltext.py b/chicago/management/commands/populate_fulltext.py new file mode 100644 index 0000000..ebaae53 --- /dev/null +++ b/chicago/management/commands/populate_fulltext.py @@ -0,0 +1,11 @@ +from django.core.management.base import BaseCommand + + +class Command(BaseCommand): + """Populate SearchBill fulltext index""" + + help = "Populate SearchBill fulltext index" + + def handle(self, *args, **kwargs): + + self.stdout.write("SearchBill populated!") diff --git a/chicago/migrations/0006_searchbill.py b/chicago/migrations/0006_searchbill.py new file mode 100644 index 0000000..38f8724 --- /dev/null +++ b/chicago/migrations/0006_searchbill.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.25 on 2024-11-22 21:59 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('chicago', '0005_chicagopersonstatistic_legislation_success_rate'), + ] + + operations = [ + migrations.CreateModel( + name='SearchBill', + fields=[ + ('text', models.TextField()), + ('last_updated', models.DateTimeField(auto_now_add=True)), + ('bill', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='chicago.chicagobill')), + ], + ), + ] diff --git a/chicago/models.py b/chicago/models.py index e7b5f7d..0851c44 100644 --- a/chicago/models.py +++ b/chicago/models.py @@ -137,6 +137,12 @@ def full_text_doc_url(self): return None +class SearchBill(models.Model): + text = models.TextField() + last_updated = models.DateTimeField(auto_now_add=True, blank=True) + bill = models.OneToOneField(ChicagoBill, on_delete=models.CASCADE, primary_key=True) + + class ChicagoOrganization(Organization): class Meta: proxy = True From 30be0fd4a57abc939ea77fdc4f0e91da4abd1a55 Mon Sep 17 00:00:00 2001 From: Derek Eder Date: Fri, 6 Dec 2024 15:55:59 -0600 Subject: [PATCH 2/3] working through using hasytack update to populate new fts index --- chicago/settings.py | 5 ++--- docker-compose.yml | 29 ++++++++++++++--------------- scripts/docker-entrypoint.sh | 2 ++ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/chicago/settings.py b/chicago/settings.py index 1596711..7c1737b 100644 --- a/chicago/settings.py +++ b/chicago/settings.py @@ -48,9 +48,8 @@ HAYSTACK_CONNECTIONS = {} HAYSTACK_CONNECTIONS["default"] = { - "ENGINE": "haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine", - "URL": os.getenv("HAYSTACK_URL", "http://elasticsearch:9200"), - "INDEX_NAME": "chicago", + "ENGINE": "postgres_fts_backend.PostgresFTSEngine", + "INDEX_NAME": "chicago_fts", "SILENTLY_FAIL": False, "BATCH_SIZE": 10, } diff --git a/docker-compose.yml b/docker-compose.yml index 12ae4c8..17a80c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,10 +14,9 @@ services: depends_on: postgres: condition: service_healthy - elasticsearch: - condition: service_started volumes: - .:/app + - /Users/derekeder/projects/datamade/postgres-fts-backend:/postgres-fts-backend environment: DJANGO_MANAGEPY_MIGRATE: "False" DJANGO_DEBUG: "True" @@ -40,19 +39,19 @@ services: ports: - 32001:5432 - elasticsearch: - image: elasticsearch:7.14.2 - container_name: chi-councilmatic-elasticsearch - ports: - - 9200:9200 - environment: - - discovery.type=single-node - - logger.org.elasticsearch.discovery=DEBUG - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - mem_limit: 1g - volumes: - - chi-councilmatic-es-data:/usr/share/elasticsearch/data + # elasticsearch: + # image: elasticsearch:7.14.2 + # container_name: chi-councilmatic-elasticsearch + # ports: + # - 9200:9200 + # environment: + # - discovery.type=single-node + # - logger.org.elasticsearch.discovery=DEBUG + # - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + # mem_limit: 1g + # volumes: + # - chi-councilmatic-es-data:/usr/share/elasticsearch/data volumes: chi-councilmatic-db-data: - chi-councilmatic-es-data: + # chi-councilmatic-es-data: diff --git a/scripts/docker-entrypoint.sh b/scripts/docker-entrypoint.sh index 427113b..a999a41 100755 --- a/scripts/docker-entrypoint.sh +++ b/scripts/docker-entrypoint.sh @@ -1,6 +1,8 @@ #!/bin/sh set -e +pip install -e /postgres-fts-backend + if [ "$DJANGO_MANAGEPY_MIGRATE" = "True" ]; then python manage.py migrate --noinput fi From 17de700f0c32bfb7645b1bd27d007abe8b610a14 Mon Sep 17 00:00:00 2001 From: Derek Eder Date: Fri, 13 Dec 2024 16:11:33 -0600 Subject: [PATCH 3/3] add extras to SearchBill --- .../management/commands/populate_fulltext.py | 11 ----------- chicago/migrations/0007_searchbill_extras.py | 18 ++++++++++++++++++ chicago/models.py | 1 + 3 files changed, 19 insertions(+), 11 deletions(-) delete mode 100644 chicago/management/commands/populate_fulltext.py create mode 100644 chicago/migrations/0007_searchbill_extras.py diff --git a/chicago/management/commands/populate_fulltext.py b/chicago/management/commands/populate_fulltext.py deleted file mode 100644 index ebaae53..0000000 --- a/chicago/management/commands/populate_fulltext.py +++ /dev/null @@ -1,11 +0,0 @@ -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - """Populate SearchBill fulltext index""" - - help = "Populate SearchBill fulltext index" - - def handle(self, *args, **kwargs): - - self.stdout.write("SearchBill populated!") diff --git a/chicago/migrations/0007_searchbill_extras.py b/chicago/migrations/0007_searchbill_extras.py new file mode 100644 index 0000000..61b0c04 --- /dev/null +++ b/chicago/migrations/0007_searchbill_extras.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.25 on 2024-12-13 21:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('chicago', '0006_searchbill'), + ] + + operations = [ + migrations.AddField( + model_name='searchbill', + name='extras', + field=models.JSONField(null=True), + ), + ] diff --git a/chicago/models.py b/chicago/models.py index 0851c44..ad2787a 100644 --- a/chicago/models.py +++ b/chicago/models.py @@ -140,6 +140,7 @@ def full_text_doc_url(self): class SearchBill(models.Model): text = models.TextField() last_updated = models.DateTimeField(auto_now_add=True, blank=True) + extras = models.JSONField(null=True) bill = models.OneToOneField(ChicagoBill, on_delete=models.CASCADE, primary_key=True)