WIP fulltext index

chicago/migrations/0006_searchbill.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.25 on 2024-11-22 21:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chicago', '0005_chicagopersonstatistic_legislation_success_rate'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='SearchBill',
+            fields=[
+                ('text', models.TextField()),
+                ('last_updated', models.DateTimeField(auto_now_add=True)),
+                ('bill', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='chicago.chicagobill')),
+            ],
+        ),
+    ]

chicago/migrations/0007_searchbill_extras.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.25 on 2024-12-13 21:38
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('chicago', '0006_searchbill'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchbill',
+            name='extras',
+            field=models.JSONField(null=True),
+        ),
+    ]

chicago/models.py

@@ -137,6 +137,13 @@ def full_text_doc_url(self):
             return None
 
 
+class SearchBill(models.Model):
+    text = models.TextField()
+    last_updated = models.DateTimeField(auto_now_add=True, blank=True)
+    extras = models.JSONField(null=True)
+    bill = models.OneToOneField(ChicagoBill, on_delete=models.CASCADE, primary_key=True)
+
+
 class ChicagoOrganization(Organization):
     class Meta:
         proxy = True

chicago/settings.py

@@ -48,9 +48,8 @@
 
 HAYSTACK_CONNECTIONS = {}
 HAYSTACK_CONNECTIONS["default"] = {
-    "ENGINE": "haystack.backends.elasticsearch7_backend.Elasticsearch7SearchEngine",
-    "URL": os.getenv("HAYSTACK_URL", "http://elasticsearch:9200"),
-    "INDEX_NAME": "chicago",
+    "ENGINE": "postgres_fts_backend.PostgresFTSEngine",
+    "INDEX_NAME": "chicago_fts",
     "SILENTLY_FAIL": False,
     "BATCH_SIZE": 10,
 }

docker-compose.yml

@@ -14,10 +14,9 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
-      elasticsearch:
-        condition: service_started
     volumes:
       - .:/app
+      - /Users/derekeder/projects/datamade/postgres-fts-backend:/postgres-fts-backend
     environment:
       DJANGO_MANAGEPY_MIGRATE: "False"
       DJANGO_DEBUG: "True"
@@ -40,19 +39,19 @@ services:
     ports:
       - 32001:5432
 
-  elasticsearch:
-    image: elasticsearch:7.14.2
-    container_name: chi-councilmatic-elasticsearch
-    ports:
-      - 9200:9200
-    environment:
-      - discovery.type=single-node
-      - logger.org.elasticsearch.discovery=DEBUG
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    mem_limit: 1g
-    volumes:
-      - chi-councilmatic-es-data:/usr/share/elasticsearch/data
+  # elasticsearch:
+  #   image: elasticsearch:7.14.2
+  #   container_name: chi-councilmatic-elasticsearch
+  #   ports:
+  #     - 9200:9200
+  #   environment:
+  #     - discovery.type=single-node
+  #     - logger.org.elasticsearch.discovery=DEBUG
+  #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+  #   mem_limit: 1g
+  #   volumes:
+  #     - chi-councilmatic-es-data:/usr/share/elasticsearch/data
 
 volumes:
   chi-councilmatic-db-data:
-  chi-councilmatic-es-data:
+  # chi-councilmatic-es-data:

scripts/docker-entrypoint.sh

@@ -1,6 +1,8 @@
 #!/bin/sh
 set -e
 
+pip install -e /postgres-fts-backend
+
 if [ "$DJANGO_MANAGEPY_MIGRATE" = "True" ]; then
     python manage.py migrate --noinput
 fi