Merge branch 'develop'

.env.example

@@ -3,7 +3,7 @@
 #
 # This file defines a set of (environment) variables that configure most
 # of the functionality of smbackend. In order for smbackend to read
-# this file, rename it to `config_dev.env`. As the name implies, this
+# this file, rename it to `.env`. As the name implies, this
 # file is supposed to be used only in development. For production use
 # we recommend setting the environment variables using the facilities
 # of your runtime environment.
@@ -120,11 +120,6 @@ STATIC_URL=/static/
 # reports. It is used to separate deployments within Sentry UI
 SENTRY_ENVIRONMENT=local-development-unconfigured
 
-# Settings needed for enabling Turku area:
-#ADDITIONAL_INSTALLED_APPS=smbackend_turku
-#TURKU_API_KEY=secret
-#ACCESSIBILITY_SYSTEM_ID=secret
-
 # The location of geo_search API
 GEO_SEARCH_LOCATION=https://paikkatietohaku.api.hel.fi/v1
 

Dockerfile

@@ -1,5 +1,5 @@
 # Using Ubuntu base for access to GDAL PPA
-FROM public.ecr.aws/ubuntu/ubuntu:22.04
+FROM public.ecr.aws/ubuntu/ubuntu:22.04 AS appbase
 WORKDIR /smbackend
 
 # tzdata installation requires settings frontend
@@ -27,8 +27,20 @@ RUN python manage.py collectstatic
 # Munigeo will fetch data to this directory
 RUN mkdir -p /smbackend/data && chgrp -R 0 /smbackend/data && chmod -R g+w /smbackend/data
 
+
+ENTRYPOINT ["./docker-entrypoint.sh"]
+
+# ==============================
+FROM appbase AS development
+# ==============================
+
+COPY requirements-dev.txt .
+RUN pip install --upgrade pip setuptools
+RUN pip install --no-cache-dir -r requirements-dev.txt
+
+# ==============================
+FROM appbase as production
+# ==============================
 # Openshift starts the container process with group zero and random ID
 # we mimic that here with nobody and group zero
 USER nobody:0
-
-ENTRYPOINT ["./docker-entrypoint.sh"]

README.md

@@ -11,19 +11,30 @@ This is the backend service for the Service Map UI.
 Installation with Docker Compose
 ------------
 
-First configure development environment settings as stated in `config_dev.env.example` and in `config_dev_ui.env.example`.
+First configure development environment settings as stated in `.env.example` and in `config_dev_ui.env.example`.
 
 ### Running the application
 
 Run application with `docker-compose up`
 
 This will startup and bind local postgres, servicemap backend and servicemap frontend containers.
 
+### Run migrations
+
+When building the application for the first time, migrations need to be run. This can be done with the following command:
+
+`docker-compose exec servicemap python manage.py migrate`
+
+
 ### Importing data
 
 To import data for development usage and automatically index it, run command:
 `docker-compose run servicemap maintenance_tasks all`
 
+However, this might take a while, especially on the first run. You can also import the needed data separately, but note
+that there are some dependencies between the data imports. The commands for the imports are found in
+[scripts/run_imports.sh](scripts/run_imports.sh).
+
 Installation without Docker
 ------------
 
@@ -174,6 +185,24 @@ Start Celery beat to handle scheduled periodic tasks with command:
 celery -A smbackend beat -l INFO
 ```
 
+Updating requirements
+---------------------
+
+pip-tools is used to manage requirements. To update the requirements, run:
+```
+pip-compile -U requirements.in
+pip-compile -U requirements-dev.in
+```
+
+Code formatting
+---------------------
+The code is formatted with black, flake8 and isort. To format the code, run:
+```
+isort .
+black .
+```
+
+
 Observations
 ------------
 

docker-compose.yml

@@ -14,6 +14,7 @@ services:
         build:
             context: ./
             dockerfile: ./Dockerfile
+            target: development
         command: start_django_development_server
         env_file:
             - .env

requirements-dev.txt

@@ -1,30 +1,32 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile requirements-dev.in
 #
 asttokens==2.4.1
     # via stack-data
-certifi==2024.6.2
+certifi==2024.8.30
     # via requests
-cffi==1.16.0
+cffi==1.17.1
     # via
     #   cryptography
     #   pynacl
 charset-normalizer==3.3.2
     # via requests
-cryptography==42.0.8
+cryptography==43.0.1
     # via pyjwt
 decorator==5.1.1
     # via ipython
 deprecated==1.2.14
     # via pygithub
-executing==2.0.1
+exceptiongroup==1.2.2
+    # via ipython
+executing==2.1.0
     # via stack-data
-idna==3.7
+idna==3.10
     # via requests
-ipython==8.25.0
+ipython==8.27.0
     # via -r requirements-dev.in
 jedi==0.19.1
     # via ipython
@@ -34,19 +36,19 @@ parso==0.8.4
     # via jedi
 pexpect==4.9.0
     # via ipython
-prompt-toolkit==3.0.46
+prompt-toolkit==3.0.47
     # via ipython
 ptyprocess==0.7.0
     # via pexpect
-pure-eval==0.2.2
+pure-eval==0.2.3
     # via stack-data
 pycparser==2.22
     # via cffi
-pygithub==2.3.0
+pygithub==2.4.0
     # via -r requirements-dev.in
 pygments==2.18.0
     # via ipython
-pyjwt[crypto]==2.8.0
+pyjwt[crypto]==2.9.0
     # via pygithub
 pynacl==1.5.0
     # via pygithub
@@ -60,11 +62,11 @@ traitlets==5.14.3
     # via
     #   ipython
     #   matplotlib-inline
-typing-extensions==4.12.1
+typing-extensions==4.12.2
     # via
     #   ipython
     #   pygithub
-urllib3==1.26.18
+urllib3==1.26.20
     # via
     #   -r requirements-dev.in
     #   pygithub

requirements.in

@@ -3,10 +3,12 @@ djangorestframework
 djangorestframework-jsonp
 django-filter
 django-modeltranslation
+django-two-factor-auth
+django-two-factor-auth[phonenumbers]
 flake8
 requests
 requests_cache
-git+https://github.com/City-of-Helsinki/[email protected].86#egg=django-munigeo
+git+https://github.com/City-of-Helsinki/[email protected].87#egg=django-munigeo
 pytz
 django-cors-headers
 django-extensions

requirements.txt

@@ -18,11 +18,11 @@ black==24.8.0
     # via -r requirements.in
 bmi-arcgis-restapi==2.4.9
     # via -r requirements.in
-build==1.2.1
+build==1.2.2
     # via pip-tools
-cattrs==23.2.3
+cattrs==24.1.2
     # via requests-cache
-certifi==2024.7.4
+certifi==2024.8.30
     # via
     #   requests
     #   sentry-sdk
@@ -34,16 +34,20 @@ click==8.1.7
     #   pip-tools
 coverage[toml]==7.6.1
     # via pytest-cov
-django==5.1
+django==5.1.1
     # via
     #   -r requirements.in
     #   django-cors-headers
     #   django-extensions
     #   django-filter
+    #   django-formtools
     #   django-js-asset
     #   django-modeltranslation
     #   django-munigeo
+    #   django-otp
+    #   django-phonenumber-field
     #   django-polymorphic
+    #   django-two-factor-auth
     #   djangorestframework
     #   drf-spectacular
 django-cors-headers==4.4.0
@@ -54,20 +58,28 @@ django-extensions==3.2.3
     # via -r requirements.in
 django-filter==24.3
     # via -r requirements.in
+django-formtools==2.5.1
+    # via django-two-factor-auth
 django-js-asset==2.2.0
     # via django-mptt
-django-modeltranslation==0.19.7
+django-modeltranslation==0.19.9
     # via
     #   -r requirements.in
     #   django-munigeo
 django-mptt==0.16.0
     # via
     #   -r requirements.in
     #   django-munigeo
-django-munigeo @ git+https://github.com/City-of-Helsinki/[email protected].86
+django-munigeo @ git+https://github.com/City-of-Helsinki/[email protected].87
     # via -r requirements.in
+django-otp==1.5.4
+    # via django-two-factor-auth
+django-phonenumber-field==8.0.0
+    # via django-two-factor-auth
 django-polymorphic==3.1.0
     # via -r requirements.in
+django-two-factor-auth[phonenumbers]==1.17.0
+    # via -r requirements.in
 djangorestframework==3.15.2
     # via
     #   -r requirements.in
@@ -88,7 +100,7 @@ geographiclib==2.0
     # via geopy
 geopy==2.4.1
     # via -r requirements.in
-idna==3.7
+idna==3.10
     # via requests
 inflection==0.5.1
     # via drf-spectacular
@@ -125,9 +137,11 @@ pathspec==0.12.1
     # via black
 pep8-naming==0.14.1
     # via -r requirements.in
+phonenumbers==8.13.45
+    # via django-two-factor-auth
 pip-tools==7.4.1
     # via -r requirements.in
-platformdirs==4.2.2
+platformdirs==4.3.6
     # via
     #   black
     #   requests-cache
@@ -139,26 +153,30 @@ pycodestyle==2.12.1
     # via flake8
 pyflakes==3.2.0
     # via flake8
+pypng==0.20220715.0
+    # via qrcode
 pyproject-hooks==1.1.0
     # via
     #   build
     #   pip-tools
-pytest==8.3.2
+pytest==8.3.3
     # via
     #   pytest-cov
     #   pytest-django
 pytest-cov==5.0.0
     # via -r requirements.in
-pytest-django==4.8.0
+pytest-django==4.9.0
     # via -r requirements.in
 python-dateutil==2.9.0.post0
     # via -r requirements.in
-pytz==2024.1
+pytz==2024.2
     # via -r requirements.in
 pyyaml==6.0.2
     # via
     #   django-munigeo
     #   drf-spectacular
+qrcode==7.4.2
+    # via django-two-factor-auth
 referencing==0.35.1
     # via
     #   jsonschema
@@ -178,7 +196,7 @@ rpds-py==0.20.0
     # via
     #   jsonschema
     #   referencing
-sentry-sdk==2.13.0
+sentry-sdk==2.14.0
     # via -r requirements.in
 six==1.16.0
     # via
@@ -202,13 +220,14 @@ typing-extensions==4.12.2
     #   black
     #   cattrs
     #   django-modeltranslation
+    #   qrcode
 tzdata==2024.1
     # via -r requirements.in
 uritemplate==4.1.1
     # via drf-spectacular
 url-normalize==1.4.3
     # via requests-cache
-urllib3==1.26.19
+urllib3==1.26.20
     # via
     #   -r requirements.in
     #   bmi-arcgis-restapi

services/api.py

@@ -991,6 +991,7 @@ def render(self, data, media_type=None, renderer_context=None):
         LEVEL_PARAMETER,
         UNIT_GEOMETRY_PARAMETER,
         UNIT_GEOMETRY_3D_PARAMETER,
+        BBOX_PARAMETER,
     ]
 )
 class UnitViewSet(

services/management/commands/statistical_queries/vantaa_population_forecast.json

@@ -100,7 +100,7 @@
       "selection": {
         "filter": "item",
         "values": [
-          "7"
+          "8"
         ]
       }
     },