From 6310c9b6fac5110fee90a2184f3ff2f8c06d7dcf Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 15 May 2024 18:21:27 +0300 Subject: [PATCH 01/17] Update packages --- requirements-dev.txt | 32 ++++++++-------- requirements.in | 2 +- requirements.txt | 87 +++++++++++++++++--------------------------- 3 files changed, 49 insertions(+), 72 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c5ffa75c8..61ca785b6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements-dev.in @@ -14,25 +14,23 @@ cffi==1.16.0 # pynacl charset-normalizer==3.3.2 # via requests -cryptography==42.0.3 +cryptography==42.0.7 # via pyjwt decorator==5.1.1 # via ipython deprecated==1.2.14 # via pygithub -exceptiongroup==1.2.0 - # via ipython executing==2.0.1 # via stack-data -idna==3.6 +idna==3.7 # via requests -ipython==8.21.0 +ipython==8.24.0 # via -r requirements-dev.in jedi==0.19.1 # via ipython -matplotlib-inline==0.1.6 +matplotlib-inline==0.1.7 # via ipython -parso==0.8.3 +parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython @@ -42,16 +40,14 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 # via stack-data -pycparser==2.21 +pycparser==2.22 # via cffi -pygithub==2.2.0 +pygithub==2.3.0 # via -r requirements-dev.in -pygments==2.17.2 +pygments==2.18.0 # via ipython pyjwt[crypto]==2.8.0 - # via - # pygithub - # pyjwt + # via pygithub pynacl==1.5.0 # via pygithub requests==2.31.0 @@ -60,12 +56,14 @@ six==1.16.0 # via asttokens stack-data==0.6.3 # via ipython -traitlets==5.14.1 +traitlets==5.14.3 # via # ipython # matplotlib-inline -typing-extensions==4.9.0 - # via pygithub +typing-extensions==4.11.0 + # via + # ipython + # pygithub urllib3==1.26.18 # via # -r requirements-dev.in diff --git a/requirements.in b/requirements.in index 916de754a..7addfa964 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.82#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index 169356820..7fb6c4d07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements.in # -asgiref==3.7.2 +asgiref==3.8.1 # via # django # django-cors-headers @@ -14,11 +14,11 @@ attrs==23.2.0 # jsonschema # referencing # requests-cache -black==24.2.0 +black==24.4.2 # via -r requirements.in bmi-arcgis-restapi==2.4.7 # via -r requirements.in -build==1.0.3 +build==1.2.1 # via pip-tools cattrs==23.2.3 # via requests-cache @@ -32,11 +32,9 @@ click==8.1.7 # via # black # pip-tools -coverage[toml]==7.4.1 - # via - # coverage - # pytest-cov -django==5.0.2 +coverage[toml]==7.5.1 + # via pytest-cov +django==5.0.6 # via # -r requirements.in # django-cors-headers @@ -54,7 +52,7 @@ django-environ==0.11.2 # via -r requirements.in django-extensions==3.2.3 # via -r requirements.in -django-filter==23.5 +django-filter==24.2 # via -r requirements.in django-js-asset==2.2.0 # via django-mptt @@ -66,22 +64,18 @@ django-mptt==0.16.0 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.82 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in -djangorestframework==3.14.0 +djangorestframework==3.15.1 # via # -r requirements.in # drf-spectacular djangorestframework-jsonp==1.0.2 # via -r requirements.in -drf-spectacular==0.27.1 +drf-spectacular==0.27.2 # via -r requirements.in -exceptiongroup==1.2.0 - # via - # cattrs - # pytest flake8==7.0.0 # via # -r requirements.in @@ -90,7 +84,7 @@ geographiclib==2.0 # via geopy geopy==2.4.1 # via -r requirements.in -idna==3.6 +idna==3.7 # via requests inflection==0.5.1 # via drf-spectacular @@ -100,13 +94,13 @@ isort==5.13.2 # via -r requirements.in jedi==0.19.1 # via -r requirements.in -jsonschema==4.21.1 +jsonschema==4.22.0 # via drf-spectacular jsonschema-specifications==2023.12.1 # via jsonschema libvoikko==4.3 # via -r requirements.in -lxml==5.1.0 +lxml==5.2.2 # via -r requirements.in mccabe==0.7.0 # via flake8 @@ -114,12 +108,12 @@ munch==4.0.0 # via bmi-arcgis-restapi mypy-extensions==1.0.0 # via black -packaging==23.2 +packaging==24.0 # via # black # build # pytest -parso==0.8.3 +parso==0.8.4 # via # -r requirements.in # jedi @@ -127,13 +121,13 @@ pathspec==0.12.1 # via black pep8-naming==0.13.3 # via -r requirements.in -pip-tools==7.4.0 +pip-tools==7.4.1 # via -r requirements.in -platformdirs==4.2.0 +platformdirs==4.2.2 # via # black # requests-cache -pluggy==1.4.0 +pluggy==1.5.0 # via pytest psycopg2==2.9.9 # via -r requirements.in @@ -141,29 +135,27 @@ pycodestyle==2.11.1 # via flake8 pyflakes==3.2.0 # via flake8 -pyproject-hooks==1.0.0 +pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.0.1 +pytest==8.2.0 # via # pytest-cov # pytest-django -pytest-cov==4.1.0 +pytest-cov==5.0.0 # via -r requirements.in pytest-django==4.8.0 # via -r requirements.in -python-dateutil==2.8.2 +python-dateutil==2.9.0.post0 # via -r requirements.in pytz==2024.1 - # via - # -r requirements.in - # djangorestframework + # via -r requirements.in pyyaml==6.0.1 # via # django-munigeo # drf-spectacular -referencing==0.33.0 +referencing==0.35.1 # via # jsonschema # jsonschema-specifications @@ -176,38 +168,25 @@ requests==2.31.0 # requests-mock requests-cache==1.2.0 # via -r requirements.in -requests-mock==1.11.0 +requests-mock==1.12.1 # via -r requirements.in -rpds-py==0.18.0 +rpds-py==0.18.1 # via # jsonschema # referencing -sentry-sdk==1.40.5 +sentry-sdk==2.1.1 # via -r requirements.in six==1.16.0 # via # django-munigeo # python-dateutil - # requests-mock # url-normalize -sqlparse==0.4.4 +sqlparse==0.5.0 # via django -tomli==2.0.1 - # via - # black - # build - # coverage - # pip-tools - # pyproject-hooks - # pytest -tqdm==4.66.2 +tqdm==4.66.4 # via -r requirements.in -typing-extensions==4.9.0 - # via - # asgiref - # black - # cattrs - # django-modeltranslation +typing-extensions==4.11.0 + # via django-modeltranslation tzdata==2024.1 # via -r requirements.in uritemplate==4.1.1 @@ -221,7 +200,7 @@ urllib3==1.26.18 # requests # requests-cache # sentry-sdk -wheel==0.42.0 +wheel==0.43.0 # via pip-tools whitenoise==6.6.0 # via -r requirements.in From 42c46578a0196586c3cdba1a58ce8fe7002a1a18 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Mon, 20 May 2024 14:33:28 +0300 Subject: [PATCH 02/17] Hyphenate addresses for search Also add feature to exclude words from search due to performance issues. --- services/fixtures/exclusion_words.json | 18 +++++ .../commands/index_search_columns.py | 62 ++++++++++++++--- services/migrations/0117_exclusionword.py | 37 +++++++++++ services/models/__init__.py | 2 +- services/models/search_rule.py | 13 ++++ services/search/api.py | 10 ++- services/search/constants.py | 4 +- services/search/tests/conftest.py | 29 +++++++- services/search/tests/test_api.py | 18 +++++ services/search/utils.py | 66 +++++++++++++++++-- 10 files changed, 241 insertions(+), 18 deletions(-) create mode 100644 services/fixtures/exclusion_words.json create mode 100644 services/migrations/0117_exclusionword.py diff --git a/services/fixtures/exclusion_words.json b/services/fixtures/exclusion_words.json new file mode 100644 index 000000000..dd36151f8 --- /dev/null +++ b/services/fixtures/exclusion_words.json @@ -0,0 +1,18 @@ +[ + { + "model": "services.exclusionword", + "pk": 1, + "fields": { + "word": "katu", + "language_short": "fi" + } + }, + { + "model": "services.exclusionword", + "pk": 2, + "fields": { + "word": "tie", + "language_short": "fi" + } + } +] \ No newline at end of file diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 11f9820c0..6ce9ca3be 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -1,11 +1,14 @@ import logging +from datetime import datetime, timedelta from django.contrib.postgres.search import SearchVector from django.core.management.base import BaseCommand +from django.utils import timezone from munigeo.models import Address, AdministrativeDivision from services.models import Service, ServiceNode, Unit -from services.search.utils import hyphenate +from services.search.constants import HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS +from services.search.utils import get_foreign_key_attr, hyphenate logger = logging.getLogger("services.management") @@ -27,17 +30,29 @@ def get_search_column(model, lang): return search_column -def generate_syllables(model): +def generate_syllables( + model, hyphenate_all_addresses=False, hyphenate_addresses_from=None +): """ Generates syllables for the given model. """ # Disable sending of signals model._meta.auto_created = True + save_kwargs = {} num_populated = 0 - for row in model.objects.all(): + if model.__name__ == "Address" and not hyphenate_all_addresses: + save_kwargs["skip_modified_at"] = True + if not hyphenate_addresses_from: + hyphenate_addresses_from = Address.objects.latest( + "modified_at" + ).modified_at - timedelta(days=HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS) + qs = model.objects.filter(modified_at__gte=hyphenate_addresses_from) + else: + qs = model.objects.all() + for row in qs: row.syllables_fi = [] for column in model.get_syllable_fi_columns(): - row_content = getattr(row, column, None) + row_content = get_foreign_key_attr(row, column) if row_content: # Rows might be of type str or Array, if str # cast to array by splitting. @@ -45,9 +60,10 @@ def generate_syllables(model): row_content = row_content.split() for word in row_content: syllables = hyphenate(word) - for s in syllables: - row.syllables_fi.append(s) - row.save() + if len(syllables) > 1: + for s in syllables: + row.syllables_fi.append(s) + row.save(**save_kwargs) num_populated += 1 # Enable sending of signals model._meta.auto_created = False @@ -85,13 +101,43 @@ def index_servicenodes(lang): class Command(BaseCommand): - def handle(self, *args, **kwargs): + def add_arguments(self, parser): + parser.add_argument( + "--hyphenate_addresses_from", + nargs="?", + type=str, + help="Hyphenate addresses whose modified_at timestamp starts at given timestamp YYYY-MM-DDTHH:MM:SS", + ) + + parser.add_argument( + "--hyphenate_all_addresses", + action="store_true", + help="Hyphenate all addresses", + ) + + def handle(self, *args, **options): + hyphenate_all_addresses = options.get("hyphenate_all_addresses", None) + hyphenate_addresses_from = options.get("hyphenate_addresses_from", None) + + if hyphenate_addresses_from: + try: + hyphenate_addresses_from = timezone.make_aware( + datetime.strptime(hyphenate_addresses_from, "%Y-%m-%dT%H:%M:%S") + ) + except ValueError as err: + raise ValueError(err) for lang in ["fi", "sv", "en"]: key = "search_column_%s" % lang # Only generate syllables for the finnish language if lang == "fi": logger.info(f"Generating syllables for language: {lang}.") logger.info(f"Syllables generated for {generate_syllables(Unit)} Units") + num_populated = generate_syllables( + Address, + hyphenate_all_addresses=hyphenate_all_addresses, + hyphenate_addresses_from=hyphenate_addresses_from, + ) + logger.info(f"Syllables generated for {num_populated} Addresses") logger.info( f"Syllables generated for {generate_syllables(Service)} Services" ) diff --git a/services/migrations/0117_exclusionword.py b/services/migrations/0117_exclusionword.py new file mode 100644 index 000000000..4fae5f0ad --- /dev/null +++ b/services/migrations/0117_exclusionword.py @@ -0,0 +1,37 @@ +# Generated by Django 5.0.6 on 2024-05-20 10:19 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("services", "0116_alter_unit_address_postal_full_and_more"), + ] + + operations = [ + migrations.CreateModel( + name="ExclusionWord", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("word", models.CharField(max_length=100, verbose_name="Word")), + ( + "language_short", + models.CharField(max_length=2, verbose_name="Language short"), + ), + ], + options={ + "verbose_name": "Exclusion word", + "verbose_name_plural": "Exclusion words", + "ordering": ["-id"], + }, + ), + ] diff --git a/services/models/__init__.py b/services/models/__init__.py index 59683c18b..2b74047de 100644 --- a/services/models/__init__.py +++ b/services/models/__init__.py @@ -4,7 +4,7 @@ from .keyword import Keyword from .mobility import MobilityServiceNode from .notification import Announcement, ErrorMessage -from .search_rule import ExclusionRule +from .search_rule import ExclusionRule, ExclusionWord from .service import Service, UnitServiceDetails from .service_mapping import ServiceMapping from .service_node import ServiceNode diff --git a/services/models/search_rule.py b/services/models/search_rule.py index 78c9c32b9..e1f0d8fe1 100644 --- a/services/models/search_rule.py +++ b/services/models/search_rule.py @@ -13,3 +13,16 @@ class Meta: def __str__(self): return "%s : %s" % (self.word, self.exclusion) + + +class ExclusionWord(models.Model): + word = models.CharField(max_length=100, verbose_name=_("Word")) + language_short = models.CharField(max_length=2, verbose_name=_("Language short")) + + class Meta: + ordering = ["-id"] + verbose_name = _("Exclusion word") + verbose_name_plural = _("Exclusion words") + + def __str__(self): + return self.word diff --git a/services/search/api.py b/services/search/api.py index 3d5db1548..ac858a1dd 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -27,9 +27,10 @@ from drf_spectacular.utils import extend_schema, OpenApiParameter from munigeo import api as munigeo_api from munigeo.models import Address, AdministrativeDivision -from rest_framework import serializers +from rest_framework import serializers, status from rest_framework.exceptions import ParseError from rest_framework.generics import GenericAPIView +from rest_framework.response import Response from services.api import ( TranslatedModelSerializer, @@ -60,6 +61,7 @@ get_preserved_order, get_service_node_results, get_trigram_results, + has_exclusion_word_in_query, set_address_fields, set_service_node_unit_count, set_service_unit_count, @@ -468,6 +470,12 @@ def get(self, request): else: search_query_str = f"{q}:*" + if has_exclusion_word_in_query(q_vals, language_short): + return Response( + f"Search query {q_vals} would return too many results", + status=status.HTTP_400_BAD_REQUEST, + ) + search_fn = "to_tsquery" if use_websearch: exclusions = self.get_search_exclusions(q) diff --git a/services/search/constants.py b/services/search/constants.py index 9d22fdaed..1d63d3b99 100644 --- a/services/search/constants.py +++ b/services/search/constants.py @@ -11,9 +11,11 @@ "Address", ) QUERY_PARAM_TYPE_NAMES = [m.lower() for m in SEARCHABLE_MODEL_TYPE_NAMES] -# None will slice to the end of list, e.g. no limit. +# None will slice to the end of list, i.e. no limit. DEFAULT_MODEL_LIMIT_VALUE = None # The limit value for the search query that search the search_view. "NULL" = no limit DEFAULT_SEARCH_SQL_LIMIT_VALUE = "NULL" DEFAULT_TRIGRAM_THRESHOLD = 0.15 DEFAULT_RANK_THRESHOLD = 0 + +HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS = 7 diff --git a/services/search/tests/conftest.py b/services/search/tests/conftest.py index 1010de99e..9cfa6167b 100644 --- a/services/search/tests/conftest.py +++ b/services/search/tests/conftest.py @@ -12,7 +12,10 @@ ) from rest_framework.test import APIClient -from services.management.commands.index_search_columns import get_search_column +from services.management.commands.index_search_columns import ( + generate_syllables, + get_search_column, +) from services.management.commands.services_import.services import ( update_service_counts, update_service_node_counts, @@ -20,6 +23,8 @@ ) from services.models import ( Department, + ExclusionRule, + ExclusionWord, Service, ServiceNode, Unit, @@ -243,6 +248,15 @@ def addresses(streets, municipality): number=1, full_name="Tarkk'ampujankatu 1", ) + Address.objects.create( + municipality_id=municipality.id, + location=Point(60.44879002342721, 22.283629416961055), + id=7, + street_id=46, + number=1, + full_name="Kellonsoittajankatu 1", + ) + generate_syllables(Address) Address.objects.update(search_column_fi=get_search_column(Address, "fi")) return Address.objects.all() @@ -280,4 +294,17 @@ def streets(): Street.objects.create(id=43, name="Markulantie", municipality_id="helsinki") Street.objects.create(id=44, name="Yliopistonkatu", municipality_id="helsinki") Street.objects.create(id=45, name="Tarkk'ampujankatu", municipality_id="helsinki") + Street.objects.create(id=46, name="Kellonsoittajankatu", municipality_id="helsinki") return Street.objects.all() + + +@pytest.fixture +def exclusion_rules(): + ExclusionRule.objects.create(id=1, word="tekojää", exclusion="-nurmi") + return ExclusionRule.objects.all() + + +@pytest.fixture +def exclusion_words(): + ExclusionWord.objects.create(id=1, word="katu", language_short="fi") + return ExclusionWord.objects.all() diff --git a/services/search/tests/test_api.py b/services/search/tests/test_api.py index 8deb6c597..67b2b82d8 100644 --- a/services/search/tests/test_api.py +++ b/services/search/tests/test_api.py @@ -13,6 +13,8 @@ def test_search( administrative_division, accessibility_shortcoming, municipality, + exclusion_rules, + exclusion_words, ): # Search for "museo" in entities: units,services and servicenods url = reverse("search") + "?q=museo&type=unit,service,servicenode" @@ -120,6 +122,22 @@ def test_search( assert kurrapolku["location"]["type"] == "Point" assert kurrapolku["location"]["coordinates"][0] == 60.479032 assert kurrapolku["location"]["coordinates"][1] == 22.25417 + # Test search with excluded word + url = reverse("search") + "?q=katu" + response = api_client.get(url) + assert response.status_code == 400 + url = reverse("search") + "?q=Katu" + response = api_client.get(url) + assert response.status_code == 400 + url = reverse("search") + "?q=koti katu" + response = api_client.get(url) + assert response.status_code == 400 + # Test search with 'kello' + url = reverse("search") + "?q=kello&type=address" + response = api_client.get(url) + results = response.json()["results"] + assert len(results) == 1 + assert results[0]["name"]["fi"] == "Kellonsoittajankatu 1" # Test address search with apostrophe in query url = reverse("search") + "?q=tarkk'ampujankatu&type=address" response = api_client.get(url) diff --git a/services/search/utils.py b/services/search/utils.py index 6790c203d..8928c85a3 100644 --- a/services/search/utils.py +++ b/services/search/utils.py @@ -1,17 +1,44 @@ +import logging + import libvoikko from django.db import connection from django.db.models import Case, When +from django.db.models.functions import Lower +from rest_framework.exceptions import ParseError -from services.models import ServiceNode, ServiceNodeUnitCount, Unit +from services.models import ( + ExclusionRule, + ExclusionWord, + ServiceNode, + ServiceNodeUnitCount, + Unit, +) from services.search.constants import ( DEFAULT_TRIGRAM_THRESHOLD, SEARCHABLE_MODEL_TYPE_NAMES, ) +logger = logging.getLogger("search") voikko = libvoikko.Voikko("fi") voikko.setNoUglyHyphenation(True) +def get_foreign_key_attr(obj, field): + """Get attr recursively by following foreign key relations + For example: + get_foreign_key_attr( + , "street__name_fi" + ) + """ + fields = field.split("__") + if len(fields) == 1: + return getattr(obj, fields[0], None) + else: + first_field = fields[0] + remaining_fields = "__".join(fields[1:]) + return get_foreign_key_attr(getattr(obj, first_field), remaining_fields) + + def is_compound_word(word): result = voikko.analyze(word) if len(result) == 0: @@ -21,7 +48,7 @@ def is_compound_word(word): def hyphenate(word): """ - Returns a list of syllables of the word if it is a compound word. + Returns a list of syllables of the word, if it is a compound word. """ word = word.strip() if is_compound_word(word): @@ -199,15 +226,42 @@ def get_preserved_order(ids): def get_trigram_results( model, model_name, field, q_val, threshold=DEFAULT_TRIGRAM_THRESHOLD ): - sql = f"""SELECT id, similarity({field}, '{q_val}') AS sml + sql = f"""SELECT id, similarity({field}, %s) AS sml FROM {model_name} - WHERE similarity({field}, '{q_val}') >= {threshold} + WHERE similarity({field}, %s) >= {threshold} ORDER BY sml DESC; """ cursor = connection.cursor() - cursor.execute(sql) + try: + cursor.execute(sql, [q_val, q_val]) + except Exception as e: + logger.error(f"Error in similarity query: {e}") + raise ParseError("Similarity query failed.") all_results = cursor.fetchall() - ids = [row[0] for row in all_results] objs = model.objects.filter(id__in=ids) return objs + + +def get_search_exclusions(q): + """ + To add/modify search exclusion rules edit: services/fixtures/exclusion_rules + To import rules: ./manage.py loaddata services/fixtures/exclusion_rules.json + """ + rule = ExclusionRule.objects.filter(word__iexact=q).first() + if rule: + return rule.exclusion + return "" + + +def has_exclusion_word_in_query(q_vals, language_short): + """ + To add/modify search exclusion words edit: services/fixtures/exclusion_words.json + To import words: ./manage.py loaddata services/fixtures/exclusion_words.json + """ + return ( + ExclusionWord.objects.filter(language_short=language_short) + .annotate(word_lower=Lower("word")) + .filter(word_lower__in=[q.lower() for q in q_vals]) + .exists() + ) From 5adc9a0f3e30f643e5011aeafc7798c9fa40142e Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Tue, 4 Jun 2024 08:50:42 +0300 Subject: [PATCH 03/17] Update only syllables-field --- services/management/commands/index_search_columns.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 6ce9ca3be..516f8453f 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -38,10 +38,8 @@ def generate_syllables( """ # Disable sending of signals model._meta.auto_created = True - save_kwargs = {} num_populated = 0 if model.__name__ == "Address" and not hyphenate_all_addresses: - save_kwargs["skip_modified_at"] = True if not hyphenate_addresses_from: hyphenate_addresses_from = Address.objects.latest( "modified_at" @@ -63,7 +61,7 @@ def generate_syllables( if len(syllables) > 1: for s in syllables: row.syllables_fi.append(s) - row.save(**save_kwargs) + row.save(update_fields=["syllables_fi"]) num_populated += 1 # Enable sending of signals model._meta.auto_created = False From b607590f60a96d52f90e55d0f4f9d1ba61c7ac5b Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 5 Jun 2024 07:57:39 +0300 Subject: [PATCH 04/17] Add more logging to syllable generation --- services/management/commands/index_search_columns.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 516f8453f..f294f1217 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -48,6 +48,7 @@ def generate_syllables( else: qs = model.objects.all() for row in qs: + logger.info(f"Generating syllables for {row}") row.syllables_fi = [] for column in model.get_syllable_fi_columns(): row_content = get_foreign_key_attr(row, column) @@ -61,7 +62,9 @@ def generate_syllables( if len(syllables) > 1: for s in syllables: row.syllables_fi.append(s) + logger.info(f"Hyphenated {word} to {syllables}") row.save(update_fields=["syllables_fi"]) + logger.info(f"Saved syllables for {row}") num_populated += 1 # Enable sending of signals model._meta.auto_created = False From 1c48973e3117831dee87c430a00e9c2e9088c8eb Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 5 Jun 2024 09:44:49 +0300 Subject: [PATCH 05/17] Trace syllable generation memory allocations --- .../commands/index_search_columns.py | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index f294f1217..a6734c39a 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -1,4 +1,5 @@ import logging +import tracemalloc from datetime import datetime, timedelta from django.contrib.postgres.search import SearchVector @@ -48,7 +49,6 @@ def generate_syllables( else: qs = model.objects.all() for row in qs: - logger.info(f"Generating syllables for {row}") row.syllables_fi = [] for column in model.get_syllable_fi_columns(): row_content = get_foreign_key_attr(row, column) @@ -62,9 +62,7 @@ def generate_syllables( if len(syllables) > 1: for s in syllables: row.syllables_fi.append(s) - logger.info(f"Hyphenated {word} to {syllables}") row.save(update_fields=["syllables_fi"]) - logger.info(f"Saved syllables for {row}") num_populated += 1 # Enable sending of signals model._meta.auto_created = False @@ -117,6 +115,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): + tracemalloc.start() hyphenate_all_addresses = options.get("hyphenate_all_addresses", None) hyphenate_addresses_from = options.get("hyphenate_addresses_from", None) @@ -131,14 +130,30 @@ def handle(self, *args, **options): key = "search_column_%s" % lang # Only generate syllables for the finnish language if lang == "fi": + snapshot1 = tracemalloc.take_snapshot() logger.info(f"Generating syllables for language: {lang}.") + logger.info("Generating syllables for Units") logger.info(f"Syllables generated for {generate_syllables(Unit)} Units") + snapshot2 = tracemalloc.take_snapshot() + + top_stats = snapshot2.compare_to(snapshot1, "lineno") + print("[ Top 10 differences ]") + for stat in top_stats[:10]: + print(stat) + + logger.info("Generating syllables for Addresses") num_populated = generate_syllables( Address, hyphenate_all_addresses=hyphenate_all_addresses, hyphenate_addresses_from=hyphenate_addresses_from, ) logger.info(f"Syllables generated for {num_populated} Addresses") + snapshot3 = tracemalloc.take_snapshot() + top_stats = snapshot3.compare_to(snapshot2, "lineno") + print("[ Top 10 differences ]") + for stat in top_stats[:10]: + print(stat) + logger.info( f"Syllables generated for {generate_syllables(Service)} Services" ) From 25e01be1c59956b213be6e4c78a1c8879a7d2db5 Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Wed, 5 Jun 2024 10:28:25 +0300 Subject: [PATCH 06/17] Limit queryset for testing purposes --- services/management/commands/index_search_columns.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index a6734c39a..2876adc0c 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -47,7 +47,7 @@ def generate_syllables( ).modified_at - timedelta(days=HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS) qs = model.objects.filter(modified_at__gte=hyphenate_addresses_from) else: - qs = model.objects.all() + qs = model.objects.all()[:100] for row in qs: row.syllables_fi = [] for column in model.get_syllable_fi_columns(): From 47080307ef8eb9900e0994167b33517a181e962c Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 7 Jun 2024 08:35:22 +0300 Subject: [PATCH 07/17] Update packages --- requirements-dev.txt | 12 ++++++------ requirements.in | 2 +- requirements.txt | 16 +++++++--------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 61ca785b6..b034df310 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ # asttokens==2.4.1 # via stack-data -certifi==2024.2.2 +certifi==2024.6.2 # via requests cffi==1.16.0 # via @@ -14,7 +14,7 @@ cffi==1.16.0 # pynacl charset-normalizer==3.3.2 # via requests -cryptography==42.0.7 +cryptography==42.0.8 # via pyjwt decorator==5.1.1 # via ipython @@ -24,7 +24,7 @@ executing==2.0.1 # via stack-data idna==3.7 # via requests -ipython==8.24.0 +ipython==8.25.0 # via -r requirements-dev.in jedi==0.19.1 # via ipython @@ -34,7 +34,7 @@ parso==0.8.4 # via jedi pexpect==4.9.0 # via ipython -prompt-toolkit==3.0.43 +prompt-toolkit==3.0.46 # via ipython ptyprocess==0.7.0 # via pexpect @@ -50,7 +50,7 @@ pyjwt[crypto]==2.8.0 # via pygithub pynacl==1.5.0 # via pygithub -requests==2.31.0 +requests==2.32.3 # via pygithub six==1.16.0 # via asttokens @@ -60,7 +60,7 @@ traitlets==5.14.3 # via # ipython # matplotlib-inline -typing-extensions==4.11.0 +typing-extensions==4.12.1 # via # ipython # pygithub diff --git a/requirements.in b/requirements.in index 7addfa964..0aa41dab3 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index 7fb6c4d07..032d1c3b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ build==1.2.1 # via pip-tools cattrs==23.2.3 # via requests-cache -certifi==2024.2.2 +certifi==2024.6.2 # via # requests # sentry-sdk @@ -32,7 +32,7 @@ click==8.1.7 # via # black # pip-tools -coverage[toml]==7.5.1 +coverage[toml]==7.5.3 # via pytest-cov django==5.0.6 # via @@ -64,7 +64,7 @@ django-mptt==0.16.0 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.83 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in @@ -119,7 +119,7 @@ parso==0.8.4 # jedi pathspec==0.12.1 # via black -pep8-naming==0.13.3 +pep8-naming==0.14.1 # via -r requirements.in pip-tools==7.4.1 # via -r requirements.in @@ -139,7 +139,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.2.0 +pytest==8.2.2 # via # pytest-cov # pytest-django @@ -159,7 +159,7 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -requests==2.31.0 +requests==2.32.3 # via # -r requirements.in # bmi-arcgis-restapi @@ -174,7 +174,7 @@ rpds-py==0.18.1 # via # jsonschema # referencing -sentry-sdk==2.1.1 +sentry-sdk==2.5.0 # via -r requirements.in six==1.16.0 # via @@ -185,8 +185,6 @@ sqlparse==0.5.0 # via django tqdm==4.66.4 # via -r requirements.in -typing-extensions==4.11.0 - # via django-modeltranslation tzdata==2024.1 # via -r requirements.in uritemplate==4.1.1 From 14ecd621ea72f11d032e37099c6420c5540ab2bd Mon Sep 17 00:00:00 2001 From: Mika Hietanen Date: Fri, 7 Jun 2024 08:35:54 +0300 Subject: [PATCH 08/17] Use Queryset.iterator() when generating syllables --- .../commands/index_search_columns.py | 28 ++++--------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/services/management/commands/index_search_columns.py b/services/management/commands/index_search_columns.py index 2876adc0c..85fc05ba8 100644 --- a/services/management/commands/index_search_columns.py +++ b/services/management/commands/index_search_columns.py @@ -1,5 +1,4 @@ import logging -import tracemalloc from datetime import datetime, timedelta from django.contrib.postgres.search import SearchVector @@ -39,16 +38,18 @@ def generate_syllables( """ # Disable sending of signals model._meta.auto_created = True + save_kwargs = {} num_populated = 0 if model.__name__ == "Address" and not hyphenate_all_addresses: + save_kwargs["skip_modified_at"] = True if not hyphenate_addresses_from: hyphenate_addresses_from = Address.objects.latest( "modified_at" ).modified_at - timedelta(days=HYPHENATE_ADDRESSES_MODIFIED_WITHIN_DAYS) qs = model.objects.filter(modified_at__gte=hyphenate_addresses_from) else: - qs = model.objects.all()[:100] - for row in qs: + qs = model.objects.all() + for row in qs.iterator(chunk_size=10000): row.syllables_fi = [] for column in model.get_syllable_fi_columns(): row_content = get_foreign_key_attr(row, column) @@ -62,7 +63,7 @@ def generate_syllables( if len(syllables) > 1: for s in syllables: row.syllables_fi.append(s) - row.save(update_fields=["syllables_fi"]) + row.save(**save_kwargs) num_populated += 1 # Enable sending of signals model._meta.auto_created = False @@ -115,8 +116,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - tracemalloc.start() - hyphenate_all_addresses = options.get("hyphenate_all_addresses", None) + hyphenate_all_addresses = options.get("hyphenate_all_addresses", False) hyphenate_addresses_from = options.get("hyphenate_addresses_from", None) if hyphenate_addresses_from: @@ -130,30 +130,14 @@ def handle(self, *args, **options): key = "search_column_%s" % lang # Only generate syllables for the finnish language if lang == "fi": - snapshot1 = tracemalloc.take_snapshot() logger.info(f"Generating syllables for language: {lang}.") - logger.info("Generating syllables for Units") logger.info(f"Syllables generated for {generate_syllables(Unit)} Units") - snapshot2 = tracemalloc.take_snapshot() - - top_stats = snapshot2.compare_to(snapshot1, "lineno") - print("[ Top 10 differences ]") - for stat in top_stats[:10]: - print(stat) - - logger.info("Generating syllables for Addresses") num_populated = generate_syllables( Address, hyphenate_all_addresses=hyphenate_all_addresses, hyphenate_addresses_from=hyphenate_addresses_from, ) logger.info(f"Syllables generated for {num_populated} Addresses") - snapshot3 = tracemalloc.take_snapshot() - top_stats = snapshot3.compare_to(snapshot2, "lineno") - print("[ Top 10 differences ]") - for stat in top_stats[:10]: - print(stat) - logger.info( f"Syllables generated for {generate_syllables(Service)} Services" ) From 185d960310c8f2ffd1af9962e41a3e2a613b2da4 Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Tue, 6 Aug 2024 13:46:48 +0300 Subject: [PATCH 09/17] Change ubuntu image source to ECR Change the ubuntu image source to Amazon ECR to avoid hitting the Docker Hub pull rate limit. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3e2bbbbf9..25efd3d76 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Using Ubuntu base for access to GDAL PPA -FROM ubuntu:22.04 +FROM public.ecr.aws/ubuntu/ubuntu:22.04 WORKDIR /smbackend # tzdata installation requires settings frontend From a3a8affdf346074c4aa6696a16347ebb8826842a Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Thu, 8 Aug 2024 12:31:35 +0300 Subject: [PATCH 10/17] Change error level to warning in department import If parent department is not found, change the error level to warning. --- services/management/commands/services_import/departments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/management/commands/services_import/departments.py b/services/management/commands/services_import/departments.py index 2c0e8a831..0448a1d80 100644 --- a/services/management/commands/services_import/departments.py +++ b/services/management/commands/services_import/departments.py @@ -48,7 +48,7 @@ def import_departments(noop=False, logger=None, fetch_resource=pk_get): parent = Department.objects.get(uuid=parent_id) obj.parent_id = parent.id except Department.DoesNotExist: - logger and logger.error( + logger and logger.warning( "Department import: no parent with uuid {} found for {}".format( parent_id, d["id"] ) From 79b80b275b5710feca16686246d972e6f00eb281 Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Thu, 8 Aug 2024 12:49:01 +0300 Subject: [PATCH 11/17] Change error level to warning in lipas import 3D Change the error level from error to warning in lipas 3D import when the unit is not saved due to a missing z coordinate. --- services/management/commands/lipas_import_3d.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/management/commands/lipas_import_3d.py b/services/management/commands/lipas_import_3d.py index b764efa04..e74e3c497 100644 --- a/services/management/commands/lipas_import_3d.py +++ b/services/management/commands/lipas_import_3d.py @@ -30,7 +30,7 @@ def _save_geometries(self, geometries, units_by_lipas_id): unit.geometry_3d = geometry unit.save() else: - logger.error( + logger.warning( f"Failed to save unit {unit.name_fi} because of a missing z coordinate.", ) From c428efded0b22bc663dec0a349ce8bd4bfb3c540 Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Thu, 8 Aug 2024 15:09:08 +0300 Subject: [PATCH 12/17] Search: handle department & municipality serialization in unit results The issue was noticed when using `include=unit.department` in search raised "Object of type Department is not JSON serializable" errors. Add department and municipality serialization to the search results to fix the issue. --- services/search/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/search/api.py b/services/search/api.py index ac858a1dd..6dfd8cd9e 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -193,6 +193,12 @@ def to_representation(self, obj): representation["connections"] = UnitConnectionSerializer( obj.connections, many=True ).data + elif "department" in include_field: + representation["department"] = DepartmentSerializer( + obj.department + ).data + elif "municipality" in include_field: + representation["municipality"] = obj.municipality.id else: if hasattr(obj, include_field): representation[include_field] = getattr( From dc5898d00edad25899910803e5fc1a39913e18f5 Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Fri, 9 Aug 2024 14:13:07 +0300 Subject: [PATCH 13/17] Fix AdministrativeDivision geometry serialization in search --- services/search/api.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/services/search/api.py b/services/search/api.py index 6dfd8cd9e..bd79e8254 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -168,8 +168,12 @@ def to_representation(self, obj): if self.context["geometry"]: if hasattr(obj, "geometry"): + if isinstance(obj, AdministrativeDivision): + geometry = obj.geometry.boundary + else: + geometry = obj.geometry representation["geometry"] = munigeo_api.geom_to_json( - obj.geometry, DEFAULT_SRS + geometry, DEFAULT_SRS ) else: representation["geometry"] = None From 0eb8c7eac8a1fafcb49580285c58dc7926e80c24 Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Mon, 12 Aug 2024 11:10:40 +0300 Subject: [PATCH 14/17] Upgrade munigeo --- requirements.in | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index 0aa41dab3..d4a550383 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.85#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index 032d1c3b4..5a68f8cb8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -64,7 +64,7 @@ django-mptt==0.16.0 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.84 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.85 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in From 5e0e7ca07e0e7fc423be183f9ffb5f74ed6e86ff Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Wed, 14 Aug 2024 09:29:46 +0300 Subject: [PATCH 15/17] Update requirements Make also a fix to django-modeltranslation use, as in version 0.19.0 the fields attribute was renamed to all_fields: https://github.com/deschler/django-modeltranslation/blob/master/CHANGELOG.md#-breaking-changes --- requirements.in | 2 +- requirements.txt | 71 ++++++++++++++++++++++++++++++------------------ services/api.py | 2 +- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/requirements.in b/requirements.in index d4a550383..bc115a755 100644 --- a/requirements.in +++ b/requirements.in @@ -6,7 +6,7 @@ django-modeltranslation flake8 requests requests_cache -git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.85#egg=django-munigeo +git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.86#egg=django-munigeo pytz django-cors-headers django-extensions diff --git a/requirements.txt b/requirements.txt index 5a68f8cb8..130ee0fbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # pip-compile requirements.in @@ -8,21 +8,21 @@ asgiref==3.8.1 # via # django # django-cors-headers -attrs==23.2.0 +attrs==24.2.0 # via # cattrs # jsonschema # referencing # requests-cache -black==24.4.2 +black==24.8.0 # via -r requirements.in -bmi-arcgis-restapi==2.4.7 +bmi-arcgis-restapi==2.4.9 # via -r requirements.in build==1.2.1 # via pip-tools cattrs==23.2.3 # via requests-cache -certifi==2024.6.2 +certifi==2024.7.4 # via # requests # sentry-sdk @@ -32,9 +32,9 @@ click==8.1.7 # via # black # pip-tools -coverage[toml]==7.5.3 +coverage[toml]==7.6.1 # via pytest-cov -django==5.0.6 +django==5.1 # via # -r requirements.in # django-cors-headers @@ -46,17 +46,17 @@ django==5.0.6 # django-polymorphic # djangorestframework # drf-spectacular -django-cors-headers==4.3.1 +django-cors-headers==4.4.0 # via -r requirements.in django-environ==0.11.2 # via -r requirements.in django-extensions==3.2.3 # via -r requirements.in -django-filter==24.2 +django-filter==24.3 # via -r requirements.in django-js-asset==2.2.0 # via django-mptt -django-modeltranslation==0.18.11 +django-modeltranslation==0.19.7 # via # -r requirements.in # django-munigeo @@ -64,11 +64,11 @@ django-mptt==0.16.0 # via # -r requirements.in # django-munigeo -django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.85 +django-munigeo @ git+https://github.com/City-of-Helsinki/django-munigeo@v0.2.86 # via -r requirements.in django-polymorphic==3.1.0 # via -r requirements.in -djangorestframework==3.15.1 +djangorestframework==3.15.2 # via # -r requirements.in # drf-spectacular @@ -76,7 +76,11 @@ djangorestframework-jsonp==1.0.2 # via -r requirements.in drf-spectacular==0.27.2 # via -r requirements.in -flake8==7.0.0 +exceptiongroup==1.2.2 + # via + # cattrs + # pytest +flake8==7.1.1 # via # -r requirements.in # pep8-naming @@ -94,13 +98,13 @@ isort==5.13.2 # via -r requirements.in jedi==0.19.1 # via -r requirements.in -jsonschema==4.22.0 +jsonschema==4.23.0 # via drf-spectacular jsonschema-specifications==2023.12.1 # via jsonschema libvoikko==4.3 # via -r requirements.in -lxml==5.2.2 +lxml==5.3.0 # via -r requirements.in mccabe==0.7.0 # via flake8 @@ -108,7 +112,7 @@ munch==4.0.0 # via bmi-arcgis-restapi mypy-extensions==1.0.0 # via black -packaging==24.0 +packaging==24.1 # via # black # build @@ -131,7 +135,7 @@ pluggy==1.5.0 # via pytest psycopg2==2.9.9 # via -r requirements.in -pycodestyle==2.11.1 +pycodestyle==2.12.1 # via flake8 pyflakes==3.2.0 # via flake8 @@ -139,7 +143,7 @@ pyproject-hooks==1.1.0 # via # build # pip-tools -pytest==8.2.2 +pytest==8.3.2 # via # pytest-cov # pytest-django @@ -151,7 +155,7 @@ python-dateutil==2.9.0.post0 # via -r requirements.in pytz==2024.1 # via -r requirements.in -pyyaml==6.0.1 +pyyaml==6.0.2 # via # django-munigeo # drf-spectacular @@ -166,41 +170,54 @@ requests==2.32.3 # django-munigeo # requests-cache # requests-mock -requests-cache==1.2.0 +requests-cache==1.2.1 # via -r requirements.in requests-mock==1.12.1 # via -r requirements.in -rpds-py==0.18.1 +rpds-py==0.20.0 # via # jsonschema # referencing -sentry-sdk==2.5.0 +sentry-sdk==2.13.0 # via -r requirements.in six==1.16.0 # via # django-munigeo # python-dateutil # url-normalize -sqlparse==0.5.0 +sqlparse==0.5.1 # via django -tqdm==4.66.4 +tomli==2.0.1 + # via + # black + # build + # coverage + # pip-tools + # pytest +tqdm==4.66.5 # via -r requirements.in +typing-extensions==4.12.2 + # via + # asgiref + # black + # cattrs + # django-modeltranslation tzdata==2024.1 # via -r requirements.in uritemplate==4.1.1 # via drf-spectacular url-normalize==1.4.3 # via requests-cache -urllib3==1.26.18 +urllib3==1.26.19 # via # -r requirements.in # bmi-arcgis-restapi # requests # requests-cache # sentry-sdk -wheel==0.43.0 +wheel==0.44.0 # via pip-tools -whitenoise==6.6.0 +whitenoise==6.7.0 # via -r requirements.in # The following packages are considered to be unsafe in a requirements file: diff --git a/services/api.py b/services/api.py index bd7e3c6d0..f45a26f4f 100644 --- a/services/api.py +++ b/services/api.py @@ -140,7 +140,7 @@ def __init__(self, *args, **kwargs): self.translated_fields = [] return - self.translated_fields = trans_opts.fields.keys() + self.translated_fields = trans_opts.all_fields.keys() # Remove the pre-existing data in the bundle. for field_name in self.translated_fields: for lang in LANGUAGES: From cc253762cbf2528bd4716261b1d88e97b48e65ca Mon Sep 17 00:00:00 2001 From: Pauliina Ilmanen Date: Wed, 14 Aug 2024 14:40:10 +0300 Subject: [PATCH 16/17] Search: handle vertical lines in search query Vertical lines are meant to be used as OR operators in the search. Remove vertical lines that are not used as meant to avoid the search breaking completely. --- services/search/api.py | 4 ++++ services/search/tests/test_api.py | 24 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/services/search/api.py b/services/search/api.py index bd79e8254..9d9bd1bf2 100644 --- a/services/search/api.py +++ b/services/search/api.py @@ -466,6 +466,10 @@ def get(self, request): config_language = LANGUAGES[language_short] search_query_str = None # Used in the raw sql + # Replace multiple consecutive vertical bars with a single vertical bar to be used as an OR operator. + q_val = re.sub(r"\|+", "|", q_val) + # Remove vertical bars that are not between words to avoid errors in the query. + q_val = re.sub(r"(? Date: Tue, 27 Aug 2024 09:54:01 +0300 Subject: [PATCH 17/17] Update setup version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d1d4b408c..a3572bd37 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name="smbackend", - version="240507", + version="240508", license="AGPLv3", packages=find_packages(), include_package_data=True,