Merge pull request #130 from ricardogsilva/integrate-vector-tile-server

Integrate traefik, vector tile server and municipalities

arpav_ppcv/bootstrapper/cliapp.py

@@ -1,11 +1,17 @@
-import typer
+import json
+from pathlib import Path
 
+import geojson_pydantic
 import sqlmodel
+import typer
 from rich import print
 from sqlalchemy.exc import IntegrityError
 
 from .. import database
-from ..schemas import observations
+from ..schemas import (
+    municipalities,
+    observations,
+)
 
 from ..schemas.coverages import (
     ConfigurationParameterCreate,
@@ -26,6 +32,34 @@
 app = typer.Typer()
 
 
+@app.command("municipalities")
+def bootstrap_municipalities(ctx: typer.Context) -> None:
+    """Bootstrap Italian municipalities."""
+    data_directory = Path(__file__).parents[2] / "data"
+    municipalities_dataset = data_directory / "limits_IT_municipalities.geojson"
+    to_create = []
+    with municipalities_dataset.open() as fh:
+        municipalities_geojson = json.load(fh)
+        for idx, feature in enumerate(municipalities_geojson["features"]):
+            print(
+                f"parsing feature ({idx+1}/{len(municipalities_geojson['features'])})..."
+            )
+            props = feature["properties"]
+            mun_create = municipalities.MunicipalityCreate(
+                geom=geojson_pydantic.MultiPolygon(
+                    type="MultiPolygon", coordinates=feature["geometry"]["coordinates"]
+                ),
+                name=props["name"],
+                province_name=props["prov_name"],
+                region_name=props["reg_name"],
+            )
+            to_create.append(mun_create)
+    print(f"About to save {len(to_create)} municipalities...")
+    with sqlmodel.Session(ctx.obj["engine"]) as session:
+        database.create_many_municipalities(session, to_create)
+    print("Done!")
+
+
 @app.command("observation-variables")
 def bootstrap_observation_variables(
     ctx: typer.Context,

arpav_ppcv/config.py

@@ -162,6 +162,7 @@ class ArpavPpcvSettings(BaseSettings):  # noqa
     templates_dir: Optional[Path] = Path(__file__).parent / "webapp/templates"
     static_dir: Optional[Path] = Path(__file__).parent / "webapp/static"
     thredds_server: ThreddsServerSettings = ThreddsServerSettings()
+    martin_tile_server_base_url: str = "http://localhost:3000"
     nearest_station_radius_meters: int = 10_000
     v1_api_mount_prefix: str = "/api/v1"
     v2_api_mount_prefix: str = "/api/v2"

arpav_ppcv/database.py

@@ -20,6 +20,7 @@
 from .schemas import (
     base,
     coverages,
+    municipalities,
     observations,
 )
 
@@ -1023,6 +1024,31 @@ def list_allowed_coverage_identifiers(
     return result
 
 
+def create_many_municipalities(
+    session: sqlmodel.Session,
+    municipalities_to_create: Sequence[municipalities.MunicipalityCreate],
+) -> list[municipalities.Municipality]:
+    """Create several municipalities."""
+    db_records = []
+    for mun_create in municipalities_to_create:
+        geom = shapely.io.from_geojson(mun_create.geom.model_dump_json())
+        wkbelement = from_shape(geom)
+        db_mun = municipalities.Municipality(
+            **mun_create.model_dump(exclude={"geom"}),
+            geom=wkbelement,
+        )
+        db_records.append(db_mun)
+        session.add(db_mun)
+    try:
+        session.commit()
+    except sqlalchemy.exc.DBAPIError:
+        raise
+    else:
+        for db_record in db_records:
+            session.refresh(db_record)
+        return db_records
+
+
 def _get_total_num_records(session: sqlmodel.Session, statement):
     return session.exec(
         sqlmodel.select(sqlmodel.func.count()).select_from(statement)

arpav_ppcv/migrations/versions/9b9809ef3088_added_municipalities_table.py

@@ -0,0 +1,40 @@
+"""added municipalities table
+
+Revision ID: 9b9809ef3088
+Revises: 74ee5bc68b7e
+Create Date: 2024-06-10 18:36:54.099021
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+import sqlmodel
+from geoalchemy2 import Geometry
+
+# revision identifiers, used by Alembic.
+revision: str = '9b9809ef3088'
+down_revision: Union[str, None] = '74ee5bc68b7e'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_geospatial_table('municipality',
+    sa.Column('id', sqlmodel.sql.sqltypes.GUID(), nullable=False),
+    sa.Column('geom', Geometry(geometry_type='MULTIPOLYGON', srid=4326, spatial_index=False, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True),
+    sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('province_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.Column('region_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False),
+    sa.PrimaryKeyConstraint('id')
+    )
+    op.create_geospatial_index('idx_municipality_geom', 'municipality', ['geom'], unique=False, postgresql_using='gist', postgresql_ops={})
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_geospatial_index('idx_municipality_geom', table_name='municipality', postgresql_using='gist', column_name='geom')
+    op.drop_geospatial_table('municipality')
+    # ### end Alembic commands ###

arpav_ppcv/schemas/municipalities.py

@@ -0,0 +1,34 @@
+import uuid
+
+import geoalchemy2
+import geojson_pydantic
+import pydantic
+import sqlalchemy
+import sqlmodel
+
+from . import fields
+
+
+class Municipality(sqlmodel.SQLModel, table=True):
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
+
+    id: pydantic.UUID4 = sqlmodel.Field(default_factory=uuid.uuid4, primary_key=True)
+    geom: fields.WkbElement = sqlmodel.Field(
+        sa_column=sqlalchemy.Column(
+            geoalchemy2.Geometry(
+                srid=4326,
+                geometry_type="MULTIPOLYGON",
+                spatial_index=True,
+            )
+        )
+    )
+    name: str
+    province_name: str
+    region_name: str
+
+
+class MunicipalityCreate(sqlmodel.SQLModel):
+    geom: geojson_pydantic.MultiPolygon
+    name: str
+    province_name: str
+    region_name: str

docker/compose.dev.yaml

@@ -14,7 +14,7 @@ x-common-env: &common-env
   ARPAV_PPCV__DEBUG: true
   ARPAV_PPCV__BIND_HOST: 0.0.0.0
   ARPAV_PPCV__BIND_PORT: 5001
-  ARPAV_PPCV__PUBLIC_URL: http://localhost:5001
+  ARPAV_PPCV__PUBLIC_URL: http://localhost:8877
   ARPAV_PPCV__DB_DSN: postgresql://arpav:arpavpassword@db:5432/arpav_ppcv
   ARPAV_PPCV__TEST_DB_DSN: postgresql://arpavtest:arpavtestpassword@test-db:5432/arpav_ppcv_test
   ARPAV_PPCV__SESSION_SECRET_KEY: some-key
@@ -26,6 +26,7 @@ x-common-env: &common-env
   ARPAV_PPCV__DJANGO_APP__REDIS_DSN: redis://redis:6379
   ARPAV_PPCV__DJANGO_APP__SECRET_KEY: some-dev-key
   ARPAV_PPCV__THREDDS_SERVER__BASE_URL: http://thredds:8080/thredds
+  ARPAV_PPCV__MARTIN_TILE_SERVER_BASE_URL: http://martin:3000
 
 x-common-volumes: &common-volumes
   - type: bind
@@ -40,6 +41,18 @@ x-common-volumes: &common-volumes
 
 services:
 
+  reverse-proxy:
+    ports:
+      - target: 80
+        published: 8877
+      - target: 8080
+        published: 8878
+    command: --configFile /traefik.toml
+    volumes:
+      - type: bind
+        source: $PWD/docker/traefik/dev-config.toml
+        target: /traefik.toml
+
   webapp:
     image: *webapp-image
     environment:
@@ -148,9 +161,14 @@ services:
     ports:
       - target: 3000
         published: 3000
+    command: ["--config", "/martin-config.yaml"]
     environment:
-      DATABASE_URL: postgres://postgres:postgres@legacy-db/postgres
+      DATABASE_URL: postgres://arpav:arpavpassword@db/arpav_ppcv
       RUST_LOG: actix_web=info,martin=debug,tokio_postgres=debug
+    volumes:
+      - type: bind
+        source: /$PWD/docker/martin/config.yaml
+        target: /martin-config.yaml
 
 volumes:
   db-data:

docker/compose.staging.yaml

@@ -23,16 +23,24 @@ name: arpav-ppcv-staging
 
 services:
 
+  reverse-proxy:
+    command: --configFile /opt/traefik/traefik.toml
+    volumes:
+      - type: bind
+        source: home/arpav/docker/traefik/staging-config.toml
+        target: /opt/traefik/traefik.toml
+      - type: bind
+        source: /opt/traefik/certs
+        target: /opt/traefik/certs
+
   webapp:
     env_file:
       - *env-file-webapp
     labels:
-      - "traefik.enable=true"
       - "traefik.http.routers.arpav-backend.entrypoints=webSecure"
       - "traefik.http.routers.arpav-backend.tls=true"
       - "traefik.http.routers.arpav-backend.tls.certResolver=letsEncryptResolver"
       - "traefik.http.routers.arpav-backend.rule=Host(`arpav.geobeyond.dev`)"
-      - "traefik.http.services.arpav-backend-service.loadbalancer.server.port=5001"
     volumes:
       - type: bind
         source: $HOME/data/arpav-ppcv/datasets
@@ -59,6 +67,11 @@ services:
   martin:
     env_file:
       - *env-file-webapp
+    labels:
+      - "traefik.http.routers.martin-router.entrypoints=webSecure"
+      - "traefik.http.routers.martin-router.tls=true"
+      - "traefik.http.routers.martin-router.tls.certResolver=letsEncryptResolver"
+      - "traefik.http.routers.martin-router.rule=Host(`arpav.geobeyond.dev`)"
     restart: unless-stopped
 
   thredds:

docker/compose.yaml

@@ -41,8 +41,19 @@ name: arpav-ppcv
 
 services:
 
+  reverse-proxy:
+    image: traefik:3.0.2
+    volumes:
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+
   webapp:
     image: "ghcr.io/geobeyond/arpav-ppcv-backend/arpav-ppcv-backend:latest"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.arpav-backend-router.rule=PathRegexp(`^/(api|admin)`)"
+      - "traefik.http.services.arpav-backend-service.loadbalancer.server.port=5001"
     depends_on:
       legacy-db:
         condition: service_healthy
@@ -67,6 +78,12 @@ services:
 
   martin:
     image: 'ghcr.io/maplibre/martin:v0.13.0'
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.martin-router.rule=PathPrefix(`/vector-tiles`)"
+      - "traefik.http.services.martin-service.loadbalancer.server.port=3000"
+      - "traefik.http.middlewares.strip-martin-prefix-middleware.stripprefix.prefixes=/vector-tiles"
+      - "traefik.http.routers.martin-router.middlewares=strip-martin-prefix-middleware@docker"
     depends_on:
       db:
         condition: service_healthy

docker/martin/config.yaml

@@ -0,0 +1,33 @@
+listen_addresses: '0.0.0.0:3000'
+cache_size_mb: 512
+postgres:
+  connection_string: ${DATABASE_URL}
+  auto_publish: false
+  tables:
+
+    stations:
+      schema: public
+      table: station
+      srid: 4326
+      geometry_column: geom
+      id_column: ~
+      geometry_type: POINT
+      properties:
+        name: varchar
+        code: varchar
+        id: uuid
+        active_since: date
+        active_until: date
+
+    municipalities:
+      schema: public
+      table: municipality
+      srid: 4326
+      geometry_column: geom
+      id_column: ~
+      geometry_type: MULTIPOLYGON
+      properties:
+        id: uuid
+        name: varchar
+        province_name: varchar
+        region_name: varchar

docker/traefik/dev-config.toml

@@ -0,0 +1,24 @@
+# Static configuration file for traefik
+#
+# In this file we mostly configure providers, entrypoints and security.
+# Routers, the other major part of a traefik configuration,  form the
+# so-called 'dynamic configuration' and in this case are gotten from
+# the labels associated with the docker provider
+#
+# More info:
+#
+# https://doc.traefik.io/traefik/
+
+[accessLog]
+
+[entryPoints]
+[entryPoints.web]
+address = ":80"
+
+[providers]
+
+[providers.docker]
+exposedByDefault = false
+
+[api]
+insecure = true

docker/traefik/staging-config.toml

@@ -0,0 +1,34 @@
+# Static configuration file for traefik
+#
+# In this file we mostly configure providers, entrypoints and security.
+# Routers, the other major part of a traefik configuration,  form the
+# so-called 'dynamic configuration' and in this case are gotten from
+# the labels associated with the docker provider
+#
+# More info:
+#
+# https://doc.traefik.io/traefik/
+
+[accessLog]
+
+[entryPoints]
+[entryPoints.webSecure]
+address = ":443"
+
+[entryPoints.webSecure.forwardedHeaders]
+insecure = true
+
+[providers]
+
+[providers.docker]
+exposedByDefault = false
+
+[certificatesResolvers.letsEncryptResolver.acme]
+email = "[email protected]"
+storage = "/opt/traefik/certs/acme.json"
+
+# Default: "https://acme-v02.api.letsencrypt.org/directory"
+# the default is the production lets encrypt server
+# caServer = "https://acme-staging-v02.api.letsencrypt.org/directory"
+
+[certificatesResolvers.letsEncryptResolver.acme.tlsChallenge]