From 9a670743e534719ff803bbbdca564edb1abd75ea Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 10 Jun 2024 20:41:53 +0100 Subject: [PATCH 1/2] Added municipalities and martin config --- arpav_ppcv/bootstrapper/cliapp.py | 38 ++++++++++++- arpav_ppcv/config.py | 1 + arpav_ppcv/database.py | 26 +++++++++ ...9b9809ef3088_added_municipalities_table.py | 40 ++++++++++++++ arpav_ppcv/schemas/municipalities.py | 34 ++++++++++++ arpav_ppcv/webapp/api_v2/app.py | 8 +++ .../webapp/api_v2/routers/tileserverproxy.py | 54 +++++++++++++++++++ .../limits_IT_municipalities.geojson | 0 .../limits_IT_provinces.geojson | 0 .../places => data}/limits_IT_regions.geojson | 0 docker/compose.dev.yaml | 8 ++- docker/martin/config.yaml | 33 ++++++++++++ 12 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 arpav_ppcv/migrations/versions/9b9809ef3088_added_municipalities_table.py create mode 100644 arpav_ppcv/schemas/municipalities.py create mode 100644 arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py rename {backend/resources/places => data}/limits_IT_municipalities.geojson (100%) rename {backend/resources/places => data}/limits_IT_provinces.geojson (100%) rename {backend/resources/places => data}/limits_IT_regions.geojson (100%) create mode 100644 docker/martin/config.yaml diff --git a/arpav_ppcv/bootstrapper/cliapp.py b/arpav_ppcv/bootstrapper/cliapp.py index 25e28df4..d6b80c84 100644 --- a/arpav_ppcv/bootstrapper/cliapp.py +++ b/arpav_ppcv/bootstrapper/cliapp.py @@ -1,11 +1,17 @@ -import typer +import json +from pathlib import Path +import geojson_pydantic import sqlmodel +import typer from rich import print from sqlalchemy.exc import IntegrityError from .. import database -from ..schemas import observations +from ..schemas import ( + municipalities, + observations, +) from ..schemas.coverages import ( ConfigurationParameterCreate, @@ -26,6 +32,34 @@ app = typer.Typer() +@app.command("municipalities") +def bootstrap_municipalities(ctx: typer.Context) -> None: + """Bootstrap Italian municipalities.""" + data_directory = Path(__file__).parents[2] / "data" + municipalities_dataset = data_directory / "limits_IT_municipalities.geojson" + to_create = [] + with municipalities_dataset.open() as fh: + municipalities_geojson = json.load(fh) + for idx, feature in enumerate(municipalities_geojson["features"]): + print( + f"parsing feature ({idx+1}/{len(municipalities_geojson['features'])})..." + ) + props = feature["properties"] + mun_create = municipalities.MunicipalityCreate( + geom=geojson_pydantic.MultiPolygon( + type="MultiPolygon", coordinates=feature["geometry"]["coordinates"] + ), + name=props["name"], + province_name=props["prov_name"], + region_name=props["reg_name"], + ) + to_create.append(mun_create) + print(f"About to save {len(to_create)} municipalities...") + with sqlmodel.Session(ctx.obj["engine"]) as session: + database.create_many_municipalities(session, to_create) + print("Done!") + + @app.command("observation-variables") def bootstrap_observation_variables( ctx: typer.Context, diff --git a/arpav_ppcv/config.py b/arpav_ppcv/config.py index 4767e472..ee03cb64 100644 --- a/arpav_ppcv/config.py +++ b/arpav_ppcv/config.py @@ -162,6 +162,7 @@ class ArpavPpcvSettings(BaseSettings): # noqa templates_dir: Optional[Path] = Path(__file__).parent / "webapp/templates" static_dir: Optional[Path] = Path(__file__).parent / "webapp/static" thredds_server: ThreddsServerSettings = ThreddsServerSettings() + martin_tile_server_base_url: str = "http://localhost:3000" nearest_station_radius_meters: int = 10_000 v1_api_mount_prefix: str = "/api/v1" v2_api_mount_prefix: str = "/api/v2" diff --git a/arpav_ppcv/database.py b/arpav_ppcv/database.py index 1be6b0a9..b67065f7 100644 --- a/arpav_ppcv/database.py +++ b/arpav_ppcv/database.py @@ -20,6 +20,7 @@ from .schemas import ( base, coverages, + municipalities, observations, ) @@ -1023,6 +1024,31 @@ def list_allowed_coverage_identifiers( return result +def create_many_municipalities( + session: sqlmodel.Session, + municipalities_to_create: Sequence[municipalities.MunicipalityCreate], +) -> list[municipalities.Municipality]: + """Create several municipalities.""" + db_records = [] + for mun_create in municipalities_to_create: + geom = shapely.io.from_geojson(mun_create.geom.model_dump_json()) + wkbelement = from_shape(geom) + db_mun = municipalities.Municipality( + **mun_create.model_dump(exclude={"geom"}), + geom=wkbelement, + ) + db_records.append(db_mun) + session.add(db_mun) + try: + session.commit() + except sqlalchemy.exc.DBAPIError: + raise + else: + for db_record in db_records: + session.refresh(db_record) + return db_records + + def _get_total_num_records(session: sqlmodel.Session, statement): return session.exec( sqlmodel.select(sqlmodel.func.count()).select_from(statement) diff --git a/arpav_ppcv/migrations/versions/9b9809ef3088_added_municipalities_table.py b/arpav_ppcv/migrations/versions/9b9809ef3088_added_municipalities_table.py new file mode 100644 index 00000000..607c5338 --- /dev/null +++ b/arpav_ppcv/migrations/versions/9b9809ef3088_added_municipalities_table.py @@ -0,0 +1,40 @@ +"""added municipalities table + +Revision ID: 9b9809ef3088 +Revises: 74ee5bc68b7e +Create Date: 2024-06-10 18:36:54.099021 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from geoalchemy2 import Geometry + +# revision identifiers, used by Alembic. +revision: str = '9b9809ef3088' +down_revision: Union[str, None] = '74ee5bc68b7e' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_geospatial_table('municipality', + sa.Column('id', sqlmodel.sql.sqltypes.GUID(), nullable=False), + sa.Column('geom', Geometry(geometry_type='MULTIPOLYGON', srid=4326, spatial_index=False, from_text='ST_GeomFromEWKT', name='geometry'), nullable=True), + sa.Column('name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('province_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.Column('region_name', sqlmodel.sql.sqltypes.AutoString(), nullable=False), + sa.PrimaryKeyConstraint('id') + ) + op.create_geospatial_index('idx_municipality_geom', 'municipality', ['geom'], unique=False, postgresql_using='gist', postgresql_ops={}) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_geospatial_index('idx_municipality_geom', table_name='municipality', postgresql_using='gist', column_name='geom') + op.drop_geospatial_table('municipality') + # ### end Alembic commands ### diff --git a/arpav_ppcv/schemas/municipalities.py b/arpav_ppcv/schemas/municipalities.py new file mode 100644 index 00000000..1921a36f --- /dev/null +++ b/arpav_ppcv/schemas/municipalities.py @@ -0,0 +1,34 @@ +import uuid + +import geoalchemy2 +import geojson_pydantic +import pydantic +import sqlalchemy +import sqlmodel + +from . import fields + + +class Municipality(sqlmodel.SQLModel, table=True): + model_config = pydantic.ConfigDict(arbitrary_types_allowed=True) + + id: pydantic.UUID4 = sqlmodel.Field(default_factory=uuid.uuid4, primary_key=True) + geom: fields.WkbElement = sqlmodel.Field( + sa_column=sqlalchemy.Column( + geoalchemy2.Geometry( + srid=4326, + geometry_type="MULTIPOLYGON", + spatial_index=True, + ) + ) + ) + name: str + province_name: str + region_name: str + + +class MunicipalityCreate(sqlmodel.SQLModel): + geom: geojson_pydantic.MultiPolygon + name: str + province_name: str + region_name: str diff --git a/arpav_ppcv/webapp/api_v2/app.py b/arpav_ppcv/webapp/api_v2/app.py index 815e5255..c65e7893 100644 --- a/arpav_ppcv/webapp/api_v2/app.py +++ b/arpav_ppcv/webapp/api_v2/app.py @@ -5,6 +5,7 @@ from .routers.coverages import router as coverages_router from .routers.observations import router as observations_router from .routers.base import router as base_router +from .routers.tileserverproxy import router as tile_server_proxy_router def create_app(settings: config.ArpavPpcvSettings) -> fastapi.FastAPI: @@ -50,4 +51,11 @@ def create_app(settings: config.ArpavPpcvSettings) -> fastapi.FastAPI: "observations", ], ) + app.include_router( + tile_server_proxy_router, + prefix="/vector-tiles", + tags=[ + "vector-tiles", + ], + ) return app diff --git a/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py b/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py new file mode 100644 index 00000000..8c026a86 --- /dev/null +++ b/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py @@ -0,0 +1,54 @@ +"""A FastAPI router that proxies requests to the tileserver.""" + +import logging +from typing import ( + Annotated, +) + +import httpx +from fastapi import ( + APIRouter, + Depends, +) +from fastapi.responses import ( + JSONResponse, + Response, +) + +from ... import dependencies +from ....config import ArpavPpcvSettings + +logger = logging.getLogger(__name__) +router = APIRouter() + + +@router.get("/catalog") +async def get_vector_tiles_catalog( + settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], + http_client: Annotated[httpx.AsyncClient, Depends(dependencies.get_http_client)], +): + response = await http_client.get(f"{settings.martin_tile_server_base_url}/catalog") + return JSONResponse( + status_code=response.status_code, + content=response.json(), + ) + + +@router.get("/{layer}/{z}/{x}/{y}") +async def vector_tiles_endpoint( + settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], + http_client: Annotated[httpx.AsyncClient, Depends(dependencies.get_http_client)], + layer: str, + z: int, + x: int, + y: int, +): + """Serve vector tiles.""" + response = await http_client.get( + f"{settings.martin_tile_server_base_url}/{layer}/{z}/{x}/{y}" + ) + return Response( + status_code=response.status_code, + content=response.content, + headers=response.headers, + ) diff --git a/backend/resources/places/limits_IT_municipalities.geojson b/data/limits_IT_municipalities.geojson similarity index 100% rename from backend/resources/places/limits_IT_municipalities.geojson rename to data/limits_IT_municipalities.geojson diff --git a/backend/resources/places/limits_IT_provinces.geojson b/data/limits_IT_provinces.geojson similarity index 100% rename from backend/resources/places/limits_IT_provinces.geojson rename to data/limits_IT_provinces.geojson diff --git a/backend/resources/places/limits_IT_regions.geojson b/data/limits_IT_regions.geojson similarity index 100% rename from backend/resources/places/limits_IT_regions.geojson rename to data/limits_IT_regions.geojson diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml index c5bf712e..0a1c65b7 100644 --- a/docker/compose.dev.yaml +++ b/docker/compose.dev.yaml @@ -26,6 +26,7 @@ x-common-env: &common-env ARPAV_PPCV__DJANGO_APP__REDIS_DSN: redis://redis:6379 ARPAV_PPCV__DJANGO_APP__SECRET_KEY: some-dev-key ARPAV_PPCV__THREDDS_SERVER__BASE_URL: http://thredds:8080/thredds + ARPAV_PPCV__MARTIN_TILE_SERVER_BASE_URL: http://martin:3000 x-common-volumes: &common-volumes - type: bind @@ -148,9 +149,14 @@ services: ports: - target: 3000 published: 3000 + command: ["--config", "/martin-config.yaml"] environment: - DATABASE_URL: postgres://postgres:postgres@legacy-db/postgres + DATABASE_URL: postgres://arpav:arpavpassword@db/arpav_ppcv RUST_LOG: actix_web=info,martin=debug,tokio_postgres=debug + volumes: + - type: bind + source: /$PWD/docker/martin/config.yaml + target: /martin-config.yaml volumes: db-data: diff --git a/docker/martin/config.yaml b/docker/martin/config.yaml new file mode 100644 index 00000000..6138f868 --- /dev/null +++ b/docker/martin/config.yaml @@ -0,0 +1,33 @@ +listen_addresses: '0.0.0.0:3000' +cache_size_mb: 512 +postgres: + connection_string: ${DATABASE_URL} + auto_publish: false + tables: + + stations: + schema: public + table: station + srid: 4326 + geometry_column: geom + id_column: ~ + geometry_type: POINT + properties: + name: varchar + code: varchar + id: uuid + active_since: date + active_until: date + + municipalities: + schema: public + table: municipality + srid: 4326 + geometry_column: geom + id_column: ~ + geometry_type: MULTIPOLYGON + properties: + id: uuid + name: varchar + province_name: varchar + region_name: varchar From 6c6842c5038ea48c1413ab36066a4926d7343b33 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Tue, 11 Jun 2024 17:58:53 +0100 Subject: [PATCH 2/2] Added traefik to compose files --- arpav_ppcv/webapp/api_v2/app.py | 8 --- .../webapp/api_v2/routers/tileserverproxy.py | 54 ------------------- docker/compose.dev.yaml | 14 ++++- docker/compose.staging.yaml | 17 +++++- docker/compose.yaml | 17 ++++++ docker/traefik/dev-config.toml | 24 +++++++++ docker/traefik/staging-config.toml | 34 ++++++++++++ 7 files changed, 103 insertions(+), 65 deletions(-) delete mode 100644 arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py create mode 100644 docker/traefik/dev-config.toml create mode 100644 docker/traefik/staging-config.toml diff --git a/arpav_ppcv/webapp/api_v2/app.py b/arpav_ppcv/webapp/api_v2/app.py index c65e7893..815e5255 100644 --- a/arpav_ppcv/webapp/api_v2/app.py +++ b/arpav_ppcv/webapp/api_v2/app.py @@ -5,7 +5,6 @@ from .routers.coverages import router as coverages_router from .routers.observations import router as observations_router from .routers.base import router as base_router -from .routers.tileserverproxy import router as tile_server_proxy_router def create_app(settings: config.ArpavPpcvSettings) -> fastapi.FastAPI: @@ -51,11 +50,4 @@ def create_app(settings: config.ArpavPpcvSettings) -> fastapi.FastAPI: "observations", ], ) - app.include_router( - tile_server_proxy_router, - prefix="/vector-tiles", - tags=[ - "vector-tiles", - ], - ) return app diff --git a/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py b/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py deleted file mode 100644 index 8c026a86..00000000 --- a/arpav_ppcv/webapp/api_v2/routers/tileserverproxy.py +++ /dev/null @@ -1,54 +0,0 @@ -"""A FastAPI router that proxies requests to the tileserver.""" - -import logging -from typing import ( - Annotated, -) - -import httpx -from fastapi import ( - APIRouter, - Depends, -) -from fastapi.responses import ( - JSONResponse, - Response, -) - -from ... import dependencies -from ....config import ArpavPpcvSettings - -logger = logging.getLogger(__name__) -router = APIRouter() - - -@router.get("/catalog") -async def get_vector_tiles_catalog( - settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], - http_client: Annotated[httpx.AsyncClient, Depends(dependencies.get_http_client)], -): - response = await http_client.get(f"{settings.martin_tile_server_base_url}/catalog") - return JSONResponse( - status_code=response.status_code, - content=response.json(), - ) - - -@router.get("/{layer}/{z}/{x}/{y}") -async def vector_tiles_endpoint( - settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], - http_client: Annotated[httpx.AsyncClient, Depends(dependencies.get_http_client)], - layer: str, - z: int, - x: int, - y: int, -): - """Serve vector tiles.""" - response = await http_client.get( - f"{settings.martin_tile_server_base_url}/{layer}/{z}/{x}/{y}" - ) - return Response( - status_code=response.status_code, - content=response.content, - headers=response.headers, - ) diff --git a/docker/compose.dev.yaml b/docker/compose.dev.yaml index 0a1c65b7..0fdaf3e0 100644 --- a/docker/compose.dev.yaml +++ b/docker/compose.dev.yaml @@ -14,7 +14,7 @@ x-common-env: &common-env ARPAV_PPCV__DEBUG: true ARPAV_PPCV__BIND_HOST: 0.0.0.0 ARPAV_PPCV__BIND_PORT: 5001 - ARPAV_PPCV__PUBLIC_URL: http://localhost:5001 + ARPAV_PPCV__PUBLIC_URL: http://localhost:8877 ARPAV_PPCV__DB_DSN: postgresql://arpav:arpavpassword@db:5432/arpav_ppcv ARPAV_PPCV__TEST_DB_DSN: postgresql://arpavtest:arpavtestpassword@test-db:5432/arpav_ppcv_test ARPAV_PPCV__SESSION_SECRET_KEY: some-key @@ -41,6 +41,18 @@ x-common-volumes: &common-volumes services: + reverse-proxy: + ports: + - target: 80 + published: 8877 + - target: 8080 + published: 8878 + command: --configFile /traefik.toml + volumes: + - type: bind + source: $PWD/docker/traefik/dev-config.toml + target: /traefik.toml + webapp: image: *webapp-image environment: diff --git a/docker/compose.staging.yaml b/docker/compose.staging.yaml index 9a86550a..91997b54 100644 --- a/docker/compose.staging.yaml +++ b/docker/compose.staging.yaml @@ -23,16 +23,24 @@ name: arpav-ppcv-staging services: + reverse-proxy: + command: --configFile /opt/traefik/traefik.toml + volumes: + - type: bind + source: home/arpav/docker/traefik/staging-config.toml + target: /opt/traefik/traefik.toml + - type: bind + source: /opt/traefik/certs + target: /opt/traefik/certs + webapp: env_file: - *env-file-webapp labels: - - "traefik.enable=true" - "traefik.http.routers.arpav-backend.entrypoints=webSecure" - "traefik.http.routers.arpav-backend.tls=true" - "traefik.http.routers.arpav-backend.tls.certResolver=letsEncryptResolver" - "traefik.http.routers.arpav-backend.rule=Host(`arpav.geobeyond.dev`)" - - "traefik.http.services.arpav-backend-service.loadbalancer.server.port=5001" volumes: - type: bind source: $HOME/data/arpav-ppcv/datasets @@ -59,6 +67,11 @@ services: martin: env_file: - *env-file-webapp + labels: + - "traefik.http.routers.martin-router.entrypoints=webSecure" + - "traefik.http.routers.martin-router.tls=true" + - "traefik.http.routers.martin-router.tls.certResolver=letsEncryptResolver" + - "traefik.http.routers.martin-router.rule=Host(`arpav.geobeyond.dev`)" restart: unless-stopped thredds: diff --git a/docker/compose.yaml b/docker/compose.yaml index d6b403d6..ecb7dca0 100644 --- a/docker/compose.yaml +++ b/docker/compose.yaml @@ -41,8 +41,19 @@ name: arpav-ppcv services: + reverse-proxy: + image: traefik:3.0.2 + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + webapp: image: "ghcr.io/geobeyond/arpav-ppcv-backend/arpav-ppcv-backend:latest" + labels: + - "traefik.enable=true" + - "traefik.http.routers.arpav-backend-router.rule=PathRegexp(`^/(api|admin)`)" + - "traefik.http.services.arpav-backend-service.loadbalancer.server.port=5001" depends_on: legacy-db: condition: service_healthy @@ -67,6 +78,12 @@ services: martin: image: 'ghcr.io/maplibre/martin:v0.13.0' + labels: + - "traefik.enable=true" + - "traefik.http.routers.martin-router.rule=PathPrefix(`/vector-tiles`)" + - "traefik.http.services.martin-service.loadbalancer.server.port=3000" + - "traefik.http.middlewares.strip-martin-prefix-middleware.stripprefix.prefixes=/vector-tiles" + - "traefik.http.routers.martin-router.middlewares=strip-martin-prefix-middleware@docker" depends_on: db: condition: service_healthy diff --git a/docker/traefik/dev-config.toml b/docker/traefik/dev-config.toml new file mode 100644 index 00000000..32883508 --- /dev/null +++ b/docker/traefik/dev-config.toml @@ -0,0 +1,24 @@ +# Static configuration file for traefik +# +# In this file we mostly configure providers, entrypoints and security. +# Routers, the other major part of a traefik configuration, form the +# so-called 'dynamic configuration' and in this case are gotten from +# the labels associated with the docker provider +# +# More info: +# +# https://doc.traefik.io/traefik/ + +[accessLog] + +[entryPoints] +[entryPoints.web] +address = ":80" + +[providers] + +[providers.docker] +exposedByDefault = false + +[api] +insecure = true diff --git a/docker/traefik/staging-config.toml b/docker/traefik/staging-config.toml new file mode 100644 index 00000000..94993230 --- /dev/null +++ b/docker/traefik/staging-config.toml @@ -0,0 +1,34 @@ +# Static configuration file for traefik +# +# In this file we mostly configure providers, entrypoints and security. +# Routers, the other major part of a traefik configuration, form the +# so-called 'dynamic configuration' and in this case are gotten from +# the labels associated with the docker provider +# +# More info: +# +# https://doc.traefik.io/traefik/ + +[accessLog] + +[entryPoints] +[entryPoints.webSecure] +address = ":443" + +[entryPoints.webSecure.forwardedHeaders] +insecure = true + +[providers] + +[providers.docker] +exposedByDefault = false + +[certificatesResolvers.letsEncryptResolver.acme] +email = "francesco.bartoli@geobeyond.it" +storage = "/opt/traefik/certs/acme.json" + +# Default: "https://acme-v02.api.letsencrypt.org/directory" +# the default is the production lets encrypt server +# caServer = "https://acme-staging-v02.api.letsencrypt.org/directory" + +[certificatesResolvers.letsEncryptResolver.acme.tlsChallenge]