diff --git a/README.md b/README.md index 63f30a8..b0ea39d 100644 --- a/README.md +++ b/README.md @@ -15,10 +15,10 @@ source .venv/bin/activate python -m pip install -e . uvicorn export TEST_ENVIRONMENT=true # set this when running locally to mock redis -uvicorn titiler.xarray.main:app --reload +uvicorn titiler.multidim.main:app --reload ``` -To access the docs, visit http://127.0.0.1:8000/api.html. +To access the docs, visit . ![](https://github.com/developmentseed/titiler-xarray/assets/10407788/4368546b-5b60-4cd5-86be-fdd959374b17) ## Testing @@ -46,7 +46,6 @@ The Github Actions workflow defined in [.github/workflows/ci.yml](./.github/work * The production stack is deployed when the `main` branch is tagged, creating a new release. The production stack will deploy to a stack with an API Gateway associated with the domain prod-titiler-xarray.delta-backend.com/. * The development stack will be deployed upon pushes to the `dev` and `main` branches. The development stack will deploy to a stack with an API Gateway associated with the domain dev-titiler-xarray.delta-backend.com/. - ## New Deployments The following steps detail how to to setup and deploy the CDK stack from your local machine. @@ -94,7 +93,6 @@ The following steps detail how to to setup and deploy the CDK stack from your lo AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production ``` - **Important** In AWS Lambda environment we need to have specific version of botocore, S3FS, FSPEC and other libraries. diff --git a/pyproject.toml b/pyproject.toml index fca40c4..33b5aef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] -name = "titiler.xarray" -description = "TiTiler extension for xarray." +name = "titiler-multidim" +description = "TiTiler application extension for titiler.xarray." readme = "README.md" requires-python = ">=3.8" authors = [ @@ -25,6 +25,8 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ + "titiler.core>=0.19.0,<0.20", + "titiler.xarray>=0.19.0,<0.20", "cftime", "h5netcdf", "numpy<2.0.0", @@ -36,13 +38,12 @@ dependencies = [ "s3fs", "aiohttp", "requests", - "pydantic==2.0.2", - "titiler.core>=0.14.1,<0.15", "pydantic-settings~=2.0", + "pydantic>=2.4,<3.0", "pandas==1.5.3", "redis", - "fastapi>=0.100.0,<0.107.0", - "starlette<0.28", + "fastapi>=0.108.0,<0.109.0", + "starlette>=0.29.0,<0.33.0", ] [project.optional-dependencies] @@ -54,7 +55,10 @@ test = [ "yappi", ] dev = [ - "pre-commit" + "dask>=2023.5.0", + "ipython>=8.12.3", + "netcdf4>=1.7.2", + "pre-commit", ] debug = [ "yappi" @@ -111,13 +115,18 @@ namespace_packages = true explicit_package_bases = true [build-system] -requires = ["pdm-pep517"] -build-backend = "pdm.pep517.api" +requires = ["pdm-backend"] +build-backend = "pdm.backend" + [tool.pdm.version] source = "file" -path = "titiler/xarray/__init__.py" +path = "src/titiler/multidim/__init__.py" + + +[tool.pdm] +package-dir = "src/" [tool.pdm.build] -includes = ["titiler/xarray"] +includes = ["src/titiler/multidim/"] excludes = ["tests/", "**/.mypy_cache", "**/.DS_Store"] diff --git a/src/titiler/multidim/__init__.py b/src/titiler/multidim/__init__.py new file mode 100644 index 0000000..d8781c1 --- /dev/null +++ b/src/titiler/multidim/__init__.py @@ -0,0 +1,3 @@ +"""titiler.multidim""" + +__version__ = "0.3.0" diff --git a/src/titiler/multidim/factory.py b/src/titiler/multidim/factory.py new file mode 100644 index 0000000..f90a0ad --- /dev/null +++ b/src/titiler/multidim/factory.py @@ -0,0 +1,181 @@ +"""TiTiler.xarray factory.""" + +from typing import List, Literal, Optional, Type +from urllib.parse import urlencode + +import jinja2 +import numpy as np +from attrs import define +from fastapi import Depends, Query +from starlette.requests import Request +from starlette.responses import HTMLResponse +from starlette.templating import Jinja2Templates +from typing_extensions import Annotated + +from titiler.core.dependencies import ColorFormulaParams, DefaultDependency +from titiler.core.resources.enums import ImageType +from titiler.core.resources.responses import JSONResponse +from titiler.multidim.reader import XarrayReader +from titiler.xarray.dependencies import DatasetParams, XarrayIOParams, XarrayParams +from titiler.xarray.factory import TilerFactory as BaseTilerFactory + + +@define(kw_only=True) +class XarrayTilerFactory(BaseTilerFactory): + """Xarray Tiler Factory.""" + + reader: Type[XarrayReader] = XarrayReader + reader_dependency: Type[DefaultDependency] = XarrayParams + dataset_dependency: Type[DefaultDependency] = DatasetParams + + def register_routes(self) -> None: # noqa: C901 + """Register Info / Tiles / TileJSON endoints.""" + super().register_routes() + self.variables() + + def variables(self) -> None: + """Register /variables endpoint""" + + @self.router.get( + "/variables", + response_class=JSONResponse, + responses={200: {"description": "Return dataset's Variables."}}, + ) + def get_variables( + src_path=Depends(self.path_dependency), + io_params=Depends(XarrayIOParams), + ) -> List[str]: + """return available variables.""" + return self.reader.list_variables( + src_path=src_path, + group=io_params.group, + decode_times=io_params.decode_times, + ) + + def statistics(self) -> None: + """Register /statistics and /histogram endpoints""" + super().statistics() + + @self.router.get( + "/histogram", + response_class=JSONResponse, + responses={200: {"description": "Return histogram for this data variable"}}, + response_model_exclude_none=True, + ) + def histogram( + src_path=Depends(self.path_dependency), + reader_params=Depends(self.reader_dependency), + ): + with self.reader( + src_path=src_path, + variable=reader_params.variable, + group=reader_params.group, + decode_times=reader_params.decode_times, + datetime=reader_params.datetime, + ) as src_dst: + boolean_mask = ~np.isnan(src_dst.input) + data_values = src_dst.input.values[boolean_mask] + counts, values = np.histogram(data_values, bins=10) + counts, values = counts.tolist(), values.tolist() + buckets = list( + zip(values, [values[i + 1] for i in range(len(values) - 1)]) + ) + hist_dict = [] + for idx, bucket in enumerate(buckets): + hist_dict.append({"bucket": bucket, "value": counts[idx]}) + return hist_dict + + def map_viewer(self) -> None: + """Register /map endpoints""" + + @self.router.get("/{tileMatrixSetId}/map", response_class=HTMLResponse) + def map_viewer( + request: Request, + tileMatrixSetId: Annotated[ # type: ignore + Literal[tuple(self.supported_tms.list())], + "Identifier selecting one of the supported TileMatrixSetIds", + ], + url: Annotated[Optional[str], Query(description="Dataset URL")] = None, + variable: Annotated[ + Optional[str], + Query(description="Xarray Variable"), + ] = None, + group: Annotated[ + Optional[int], + Query( + description="Select a specific zarr group from a zarr hierarchy, can be for pyramids or datasets. Can be used to open a dataset in HDF5 files." + ), + ] = None, + decode_times: Annotated[ + bool, + Query( + title="decode_times", + description="Whether to decode times", + ), + ] = True, + drop_dim: Annotated[ + Optional[str], + Query(description="Dimension to drop"), + ] = None, + datetime: Annotated[ + Optional[str], Query(description="Slice of time to read (if available)") + ] = None, + tile_format: Annotated[ + Optional[ImageType], + Query( + description="Default will be automatically defined if the output image needs a mask (png) or not (jpeg).", + ), + ] = None, + tile_scale: Annotated[ + int, + Query( + gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..." + ), + ] = 1, + minzoom: Annotated[ + Optional[int], + Query(description="Overwrite default minzoom."), + ] = None, + maxzoom: Annotated[ + Optional[int], + Query(description="Overwrite default maxzoom."), + ] = None, + post_process=Depends(self.process_dependency), + rescale=Depends(self.rescale_dependency), + color_formula=Depends(ColorFormulaParams), + colormap=Depends(self.colormap_dependency), + render_params=Depends(self.render_dependency), + dataset_params=Depends(self.dataset_dependency), + ): + """Return map Viewer.""" + jinja2_env = jinja2.Environment( + loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, ".")]) + ) + templates = Jinja2Templates(env=jinja2_env) + + if url: + tilejson_url = self.url_for( + request, "tilejson", tileMatrixSetId=tileMatrixSetId + ) + if request.query_params._list: + tilejson_url += f"?{urlencode(request.query_params._list)}" + + tms = self.supported_tms.get(tileMatrixSetId) + return templates.TemplateResponse( + name="map.html", + context={ + "request": request, + "tilejson_endpoint": tilejson_url, + "tms": tms, + "resolutions": [matrix.cellSize for matrix in tms], + }, + media_type="text/html", + ) + else: + return templates.TemplateResponse( + name="map-form.html", + context={ + "request": request, + }, + media_type="text/html", + ) diff --git a/titiler/xarray/main.py b/src/titiler/multidim/main.py similarity index 80% rename from titiler/xarray/main.py rename to src/titiler/multidim/main.py index 65d0e2f..ee42f10 100644 --- a/titiler/xarray/main.py +++ b/src/titiler/multidim/main.py @@ -1,4 +1,4 @@ -"""titiler app.""" +"""titiler.multidim.""" import logging @@ -8,19 +8,19 @@ from starlette import status from starlette.middleware.cors import CORSMiddleware -import titiler.xarray.reader as reader +import titiler.multidim.reader as reader from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers -from titiler.core.factory import AlgorithmFactory, TMSFactory +from titiler.core.factory import AlgorithmFactory, ColorMapFactory, TMSFactory from titiler.core.middleware import ( CacheControlMiddleware, LoggerMiddleware, TotalTimeMiddleware, ) -from titiler.xarray import __version__ as titiler_version -from titiler.xarray.factory import ZarrTilerFactory -from titiler.xarray.middleware import ServerTimingMiddleware -from titiler.xarray.redis_pool import get_redis -from titiler.xarray.settings import ApiSettings +from titiler.multidim import __version__ as titiler_version +from titiler.multidim.factory import XarrayTilerFactory +from titiler.multidim.middleware import ServerTimingMiddleware +from titiler.multidim.redis_pool import get_redis +from titiler.multidim.settings import ApiSettings logging.getLogger("botocore.credentials").disabled = True logging.getLogger("botocore.utils").disabled = True @@ -38,7 +38,7 @@ ############################################################################### # Tiles endpoints -xarray_factory = ZarrTilerFactory() +xarray_factory = XarrayTilerFactory() app.include_router(xarray_factory.router, tags=["Xarray Tiler API"]) ############################################################################### @@ -51,6 +51,14 @@ algorithms = AlgorithmFactory() app.include_router(algorithms.router, tags=["Algorithms"]) +############################################################################### +# Colormaps endpoints +cmaps = ColorMapFactory() +app.include_router( + cmaps.router, + tags=["ColorMaps"], +) + error_codes = { zarr.errors.GroupNotFoundError: status.HTTP_422_UNPROCESSABLE_ENTITY, } diff --git a/titiler/xarray/map-form.html b/src/titiler/multidim/map-form.html similarity index 81% rename from titiler/xarray/map-form.html rename to src/titiler/multidim/map-form.html index 8bd800f..b83ebe9 100644 --- a/titiler/xarray/map-form.html +++ b/src/titiler/multidim/map-form.html @@ -123,14 +123,6 @@

Step 1: Enter the URL of your Zarr store


Step 2: Define other fields to use when opening the URL with xarray

- - -
-
- - -
-

If the URL is for a zarr hierarchy or HDF5, please specify the group to use when opening the dataset.
@@ -173,16 +165,28 @@

Step 4: Select a colormap.