From 525a38fb72604be1917349036d9aaf1acc299a20 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Fri, 29 Sep 2023 01:13:46 -0500 Subject: [PATCH 1/9] Updated vector layer factory to pydantic v2 --- pyproject.toml | 58 ++++++++++++++++++------------------------ timvt/db.py | 2 +- timvt/factory.py | 11 +++----- timvt/layer.py | 12 ++++----- timvt/models/mapbox.py | 2 +- timvt/settings.py | 27 ++++++++++---------- 6 files changed, 50 insertions(+), 62 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 440bde7..c23d7a5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,10 +3,10 @@ name = "timvt" description = "A lightweight PostGIS based dynamic vector tile server." readme = "README.md" requires-python = ">=3.8" -license = {file = "LICENSE"} +license = { file = "LICENSE" } authors = [ - {name = "Vincent Sarago", email = "vincent@developmentseed.org"}, - {name = "David Bitner", email = "david@developmentseed.org"}, + { name = "Vincent Sarago", email = "vincent@developmentseed.org" }, + { name = "David Bitner", email = "david@developmentseed.org" }, ] keywords = ["FastAPI", "MVT", "POSTGIS"] classifiers = [ @@ -26,9 +26,10 @@ dependencies = [ "buildpg>=0.3", "fastapi>=0.87", "jinja2>=2.11.2,<4.0.0", - "morecantile>=3.1,<4.0", + "morecantile>=5.0,<6.0", "starlette-cramjam>=0.3,<0.4", "importlib_resources>=1.1.0; python_version < '3.9'", + "pydantic-settings>=2.0.3", "typing_extensions; python_version < '3.9.2'", ] @@ -46,12 +47,8 @@ test = [ "numpy", "sqlalchemy>=1.1,<1.4", ] -dev = [ - "pre-commit", -] -server = [ - "uvicorn[standard]>=0.12.0,<0.19.0", -] +dev = ["pre-commit"] +server = ["uvicorn[standard]>=0.12.0,<0.19.0"] docs = [ "nbconvert", "mkdocs", @@ -71,22 +68,22 @@ path = "timvt/__init__.py" [tool.hatch.build.targets.sdist] exclude = [ - "/tests", - "/dockerfiles", - "/docs", - "/demo", - "/data", - "docker-compose.yml", - "CONTRIBUTING.md", - "CHANGES.md", - ".pytest_cache", - ".history", - ".github", - ".env.example", - ".bumpversion.cfg", - ".flake8", - ".gitignore", - ".pre-commit-config.yaml", + "/tests", + "/dockerfiles", + "/docs", + "/demo", + "/data", + "docker-compose.yml", + "CONTRIBUTING.md", + "CHANGES.md", + ".pytest_cache", + ".history", + ".github", + ".env.example", + ".bumpversion.cfg", + ".flake8", + ".gitignore", + ".pre-commit-config.yaml", ] [build-system] @@ -96,13 +93,8 @@ build-backend = "hatchling.build" [tool.isort] profile = "black" known_first_party = ["timvt"] -known_third_party = [ - "morecantile", -] -forced_separate = [ - "fastapi", - "starlette", -] +known_third_party = ["morecantile"] +forced_separate = ["fastapi", "starlette"] default_section = "THIRDPARTY" [tool.mypy] diff --git a/timvt/db.py b/timvt/db.py index 3c7f086..523fd48 100644 --- a/timvt/db.py +++ b/timvt/db.py @@ -31,7 +31,7 @@ async def connect_to_db( settings = PostgresSettings() app.state.pool = await asyncpg.create_pool_b( - settings.database_url, + settings.database_url.unicode_string(), min_size=settings.db_min_conn_size, max_size=settings.db_max_conn_size, max_queries=settings.db_max_queries, diff --git a/timvt/factory.py b/timvt/factory.py index bca0015..9a87f6e 100644 --- a/timvt/factory.py +++ b/timvt/factory.py @@ -15,6 +15,7 @@ from timvt.resources.enums import MimeTypes from fastapi import APIRouter, Depends, Path, Query +from fastapi.params import Param from starlette.datastructures import QueryParams from starlette.requests import Request @@ -114,10 +115,7 @@ def register_tiles(self): async def tile( request: Request, tile: Tile = Depends(TileParams), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( - self.default_tms, - description=f"TileMatrixSet Name (default: '{self.default_tms}')", - ), + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = self.default_tms, layer=Depends(self.layer_dependency), ): """Return vector tile.""" @@ -146,10 +144,7 @@ async def tile( async def tilejson( request: Request, layer=Depends(self.layer_dependency), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = Query( - self.default_tms, - description=f"TileMatrixSet Name (default: '{self.default_tms}')", - ), + TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = self.default_tms, minzoom: Optional[int] = Query( None, description="Overwrite default minzoom." ), diff --git a/timvt/layer.py b/timvt/layer.py index d3d8e19..29dab69 100644 --- a/timvt/layer.py +++ b/timvt/layer.py @@ -38,12 +38,12 @@ class Layer(BaseModel, metaclass=abc.ABCMeta): id: str bounds: List[float] = [-180, -90, 180, 90] crs: str = "http://www.opengis.net/def/crs/EPSG/0/4326" - title: Optional[str] - description: Optional[str] + title: Optional[str] = None + description: Optional[str] = None minzoom: int = tile_settings.default_minzoom maxzoom: int = tile_settings.default_maxzoom default_tms: str = tile_settings.default_tms - tileurl: Optional[str] + tileurl: Optional[str] = None @abc.abstractmethod async def get_tile( @@ -89,7 +89,7 @@ class Table(Layer, DBTable): type: str = "Table" - @root_validator + @root_validator(pre=True) def bounds_default(cls, values): """Get default bounds from the first geometry columns.""" geoms = values.get("geometry_columns") @@ -239,9 +239,9 @@ class Function(Layer): type: str = "Function" sql: str function_name: Optional[str] - options: Optional[List[Dict[str, Any]]] + options: Optional[List[Dict[str, Any]]] = None - @root_validator + @root_validator(pre=True) def function_name_default(cls, values): """Define default function's name to be same as id.""" function_name = values.get("function_name") diff --git a/timvt/models/mapbox.py b/timvt/models/mapbox.py index aa5c413..ff11abe 100644 --- a/timvt/models/mapbox.py +++ b/timvt/models/mapbox.py @@ -37,7 +37,7 @@ class TileJSON(BaseModel): bounds: List[float] = [-180, -90, 180, 90] center: Optional[Tuple[float, float, int]] - @root_validator + @root_validator(pre=True) def compute_center(cls, values): """Compute center if it does not exist.""" bounds = values["bounds"] diff --git a/timvt/settings.py b/timvt/settings.py index 7ace7e7..0d0f6be 100644 --- a/timvt/settings.py +++ b/timvt/settings.py @@ -1,19 +1,20 @@ """ TiMVT config. -TiMVT uses pydantic.BaseSettings to either get settings from `.env` or environment variables +TiMVT uses BaseSettings to either get settings from `.env` or environment variables see: https://pydantic-docs.helpmanual.io/usage/settings/ """ import sys from functools import lru_cache from typing import Any, Dict, List, Optional - +from pydantic_settings import BaseSettings import pydantic + # Pydantic does not support older versions of typing.TypedDict # https://github.com/pydantic/pydantic/pull/3374 -if sys.version_info < (3, 9, 2): +if sys.version_info < (3, 12, 0): from typing_extensions import TypedDict else: from typing import TypedDict @@ -28,7 +29,7 @@ class TableConfig(TypedDict, total=False): properties: Optional[List[str]] -class TableSettings(pydantic.BaseSettings): +class TableSettings(BaseSettings): """Table configuration settings""" fallback_key_names: List[str] = ["ogc_fid", "id", "pkey", "gid"] @@ -42,7 +43,7 @@ class Config: env_nested_delimiter = "__" -class _ApiSettings(pydantic.BaseSettings): +class _ApiSettings(BaseSettings): """API settings""" name: str = "TiMVT" @@ -76,7 +77,7 @@ def ApiSettings() -> _ApiSettings: return _ApiSettings() -class _TileSettings(pydantic.BaseSettings): +class _TileSettings(BaseSettings): """MVT settings""" tile_resolution: int = 4096 @@ -99,7 +100,7 @@ def TileSettings() -> _TileSettings: return _TileSettings() -class PostgresSettings(pydantic.BaseSettings): +class PostgresSettings(BaseSettings): """Postgres-specific API settings. Attributes: @@ -110,11 +111,11 @@ class PostgresSettings(pydantic.BaseSettings): postgres_dbname: database name. """ - postgres_user: Optional[str] - postgres_pass: Optional[str] - postgres_host: Optional[str] - postgres_port: Optional[str] - postgres_dbname: Optional[str] + postgres_user: Optional[str] = None + postgres_pass: Optional[str] = None + postgres_host: Optional[str] = None + postgres_port: Optional[str] = None + postgres_dbname: Optional[str] = None database_url: Optional[pydantic.PostgresDsn] = None @@ -124,7 +125,7 @@ class PostgresSettings(pydantic.BaseSettings): db_max_inactive_conn_lifetime: float = 300 db_schemas: List[str] = ["public"] - db_tables: Optional[List[str]] + db_tables: Optional[List[str]] = None class Config: """model config""" From 8f70717eb576e311b37a2793c57db2ddf86a9801 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Fri, 29 Sep 2023 01:16:37 -0500 Subject: [PATCH 2/9] Bumped version to 0.8.0a4 --- timvt/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timvt/__init__.py b/timvt/__init__.py index fadae60..72f9889 100644 --- a/timvt/__init__.py +++ b/timvt/__init__.py @@ -1,3 +1,3 @@ """timvt.""" -__version__ = "0.8.0a3" +__version__ = "0.8.0a4" From e5641c0c345459f96539c5ad54cc15f1ddf7fba1 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Wed, 14 Feb 2024 11:32:59 -0700 Subject: [PATCH 3/9] Update some layer queries --- timvt/dbmodel.py | 1 - timvt/layer.py | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/timvt/dbmodel.py b/timvt/dbmodel.py index 686452f..ecfcdf8 100644 --- a/timvt/dbmodel.py +++ b/timvt/dbmodel.py @@ -4,7 +4,6 @@ from buildpg import asyncpg from pydantic import BaseModel, Field - from timvt.settings import TableSettings diff --git a/timvt/layer.py b/timvt/layer.py index 29dab69..9fbd9ae 100644 --- a/timvt/layer.py +++ b/timvt/layer.py @@ -10,7 +10,6 @@ from buildpg import Var as pg_variable from buildpg import asyncpg, clauses, funcs, render, select_fields from pydantic import BaseModel, root_validator - from timvt.dbmodel import Table as DBTable from timvt.errors import ( InvalidGeometryColumnName, @@ -95,9 +94,16 @@ def bounds_default(cls, values): geoms = values.get("geometry_columns") if geoms: # Get the Extent of all the bounds - minx, miny, maxx, maxy = zip(*[geom.bounds for geom in geoms]) + def get_bounds(geom): + bounds = getattr(geom, "bounds", None) + if bounds is None: + bounds = geom["bounds"] + return bounds + + minx, miny, maxx, maxy = zip(*[get_bounds(geom) for geom in geoms]) values["bounds"] = [min(minx), min(miny), max(maxx), max(maxy)] - values["crs"] = f"http://www.opengis.net/def/crs/EPSG/0/{geoms[0].srid}" + srid = geoms[0]["srid"] + values["crs"] = f"http://www.opengis.net/def/crs/EPSG/0/{srid}" return values From 7f4ec5c9007693e37feb8ec1cef356a5db2c4883 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Wed, 1 May 2024 23:55:00 -0400 Subject: [PATCH 4/9] Hack to ensure that env file isn't checked by Pydantic --- timvt/settings.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/timvt/settings.py b/timvt/settings.py index 0d0f6be..d84c03a 100644 --- a/timvt/settings.py +++ b/timvt/settings.py @@ -5,12 +5,13 @@ see: https://pydantic-docs.helpmanual.io/usage/settings/ """ + import sys from functools import lru_cache from typing import Any, Dict, List, Optional -from pydantic_settings import BaseSettings -import pydantic +import pydantic +from pydantic_settings import BaseSettings # Pydantic does not support older versions of typing.TypedDict # https://github.com/pydantic/pydantic/pull/3374 @@ -130,7 +131,7 @@ class PostgresSettings(BaseSettings): class Config: """model config""" - env_file = ".env" + env_file = ".env-test" # https://github.com/tiangolo/full-stack-fastapi-postgresql/blob/master/%7B%7Bcookiecutter.project_slug%7D%7D/backend/app/app/core/config.py#L42 @pydantic.validator("database_url", pre=True) From 473d12d382cb100b9381fbc29fd512246acae956 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Fri, 10 May 2024 00:18:57 -0400 Subject: [PATCH 5/9] Update some table information --- timvt/db.py | 7 +++---- timvt/dbmodel.py | 4 +++- timvt/factory.py | 26 ++++++++++++++------------ timvt/settings.py | 7 ++++--- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/timvt/db.py b/timvt/db.py index 523fd48..bf9d96c 100644 --- a/timvt/db.py +++ b/timvt/db.py @@ -1,15 +1,14 @@ """timvt.db: database events.""" +from os import getenv from typing import Any, Optional import orjson from buildpg import asyncpg - +from fastapi import FastAPI from timvt.dbmodel import get_table_index from timvt.settings import PostgresSettings -from fastapi import FastAPI - async def con_init(conn): """Use json for json returns.""" @@ -31,7 +30,7 @@ async def connect_to_db( settings = PostgresSettings() app.state.pool = await asyncpg.create_pool_b( - settings.database_url.unicode_string(), + settings.database_url, min_size=settings.db_min_conn_size, max_size=settings.db_max_conn_size, max_queries=settings.db_max_queries, diff --git a/timvt/dbmodel.py b/timvt/dbmodel.py index ecfcdf8..8d4ff8c 100644 --- a/timvt/dbmodel.py +++ b/timvt/dbmodel.py @@ -236,7 +236,9 @@ async def get_table_index( jsonb_build_object( 'name', attname, 'type', "type", - 'description', description + 'description', description, + 'min', NULL, + 'max', NULL ) ) FILTER (WHERE type LIKE 'timestamp%'), '[]'::jsonb) as datetime_columns, coalesce(jsonb_agg( diff --git a/timvt/factory.py b/timvt/factory.py index 9a87f6e..2480311 100644 --- a/timvt/factory.py +++ b/timvt/factory.py @@ -4,24 +4,21 @@ from typing import Any, Callable, Dict, List, Literal, Optional from urllib.parse import urlencode +from fastapi import APIRouter, Depends, Path, Query +from fastapi.params import Param from morecantile import Tile, TileMatrixSet from morecantile import tms as morecantile_tms from morecantile.defaults import TileMatrixSets - -from timvt.dependencies import LayerParams, TileParams -from timvt.layer import Function, Layer, Table -from timvt.models.mapbox import TileJSON -from timvt.models.OGC import TileMatrixSetList -from timvt.resources.enums import MimeTypes - -from fastapi import APIRouter, Depends, Path, Query -from fastapi.params import Param - from starlette.datastructures import QueryParams from starlette.requests import Request from starlette.responses import HTMLResponse, Response from starlette.routing import NoMatchFound from starlette.templating import Jinja2Templates +from timvt.dependencies import LayerParams, TileParams +from timvt.layer import Function, Layer, Table +from timvt.models.mapbox import TileJSON +from timvt.models.OGC import TileMatrixSetList +from timvt.resources.enums import MimeTypes try: from importlib.resources import files as resources_files # type: ignore @@ -115,7 +112,9 @@ def register_tiles(self): async def tile( request: Request, tile: Tile = Depends(TileParams), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = self.default_tms, + TileMatrixSetId: Literal[ + tuple(self.supported_tms.list()) + ] = self.default_tms, layer=Depends(self.layer_dependency), ): """Return vector tile.""" @@ -144,7 +143,9 @@ async def tile( async def tilejson( request: Request, layer=Depends(self.layer_dependency), - TileMatrixSetId: Literal[tuple(self.supported_tms.list())] = self.default_tms, + TileMatrixSetId: Literal[ + tuple(self.supported_tms.list()) + ] = self.default_tms, minzoom: Optional[int] = Query( None, description="Overwrite default minzoom." ), @@ -210,6 +211,7 @@ def _get_tiles_url(id) -> Optional[str]: return None table_catalog = getattr(request.app.state, "table_catalog", {}) + print(table_catalog) return [ Table(**table_info, tileurl=_get_tiles_url(table_id)) for table_id, table_info in table_catalog.items() diff --git a/timvt/settings.py b/timvt/settings.py index 0d0f6be..b9a62ba 100644 --- a/timvt/settings.py +++ b/timvt/settings.py @@ -5,12 +5,13 @@ see: https://pydantic-docs.helpmanual.io/usage/settings/ """ + import sys from functools import lru_cache from typing import Any, Dict, List, Optional -from pydantic_settings import BaseSettings -import pydantic +import pydantic +from pydantic_settings import BaseSettings # Pydantic does not support older versions of typing.TypedDict # https://github.com/pydantic/pydantic/pull/3374 @@ -117,7 +118,7 @@ class PostgresSettings(BaseSettings): postgres_port: Optional[str] = None postgres_dbname: Optional[str] = None - database_url: Optional[pydantic.PostgresDsn] = None + database_url: Optional[str] = None db_min_conn_size: int = 1 db_max_conn_size: int = 10 From 50ff8f22977254a91314e14113086a579d66e6ca Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Fri, 10 May 2024 03:00:14 -0400 Subject: [PATCH 6/9] Made optional values more explicit --- timvt/models/mapbox.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/timvt/models/mapbox.py b/timvt/models/mapbox.py index ff11abe..130e846 100644 --- a/timvt/models/mapbox.py +++ b/timvt/models/mapbox.py @@ -23,15 +23,15 @@ class TileJSON(BaseModel): tilejson: str = "2.2.0" name: Optional[str] - description: Optional[str] + description: Optional[str] = None version: str = "1.0.0" - attribution: Optional[str] - template: Optional[str] - legend: Optional[str] + attribution: Optional[str] = None + template: Optional[str] = None + legend: Optional[str] = None scheme: SchemeEnum = SchemeEnum.xyz tiles: List[str] - grids: Optional[List[str]] - data: Optional[List[str]] + grids: Optional[List[str]] = None + data: Optional[List[str]] = None minzoom: int = Field(0, ge=0, le=30) maxzoom: int = Field(30, ge=0, le=30) bounds: List[float] = [-180, -90, 180, 90] From c99c4a79e4a84a274dcb486c2f95724f02cc02b7 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Fri, 10 May 2024 03:29:18 -0400 Subject: [PATCH 7/9] Updated column name subsetting --- timvt/layer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/timvt/layer.py b/timvt/layer.py index 9fbd9ae..1cc7596 100644 --- a/timvt/layer.py +++ b/timvt/layer.py @@ -6,7 +6,7 @@ from typing import Any, ClassVar, Dict, List, Optional import morecantile -from buildpg import Func +from buildpg import Func, RawDangerous from buildpg import Var as pg_variable from buildpg import asyncpg, clauses, funcs, render, select_fields from pydantic import BaseModel, root_validator @@ -157,6 +157,12 @@ async def get_tile( tms_srid = tms.crs.to_epsg() tms_proj = tms.crs.to_proj4() + # This may be more valid but it doesn't add quotes around the column names + # _fields = select_fields(*cols) + + _fields = [f'"{f}"' for f in cols] + _fields = ", ".join(_fields) + async with pool.acquire() as conn: sql_query = """ WITH @@ -209,7 +215,7 @@ async def get_tile( sql_query, tablename=pg_variable(self.id), geometry_column=pg_variable(geometry_column.name), - fields=select_fields(*cols), + fields=RawDangerous(_fields), xmin=bbox.left, ymin=bbox.bottom, xmax=bbox.right, From d0f8b396ca67e9b62f76e040b8e2d28104690638 Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Wed, 10 Jul 2024 23:17:12 -0500 Subject: [PATCH 8/9] Loosen requirements on environment variables --- timvt/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/timvt/settings.py b/timvt/settings.py index b9a62ba..29bbdbf 100644 --- a/timvt/settings.py +++ b/timvt/settings.py @@ -42,6 +42,7 @@ class Config: env_prefix = "TIMVT_" env_file = ".env" env_nested_delimiter = "__" + extra = "ignore" class _ApiSettings(BaseSettings): @@ -93,6 +94,7 @@ class Config: env_prefix = "TIMVT_" env_file = ".env" + extra = "ignore" @lru_cache() From b0dfae566a94fef067b83a5034ababc14c87da3d Mon Sep 17 00:00:00 2001 From: Daven Quinn Date: Thu, 14 Nov 2024 23:52:18 -0600 Subject: [PATCH 9/9] DOn't print lots of data --- timvt/factory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/timvt/factory.py b/timvt/factory.py index 2480311..ec9296e 100644 --- a/timvt/factory.py +++ b/timvt/factory.py @@ -211,7 +211,6 @@ def _get_tiles_url(id) -> Optional[str]: return None table_catalog = getattr(request.app.state, "table_catalog", {}) - print(table_catalog) return [ Table(**table_info, tileurl=_get_tiles_url(table_id)) for table_id, table_info in table_catalog.items()