Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

use factory from titiler.xarray #72

Open
wants to merge 8 commits into
base: dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ source .venv/bin/activate

python -m pip install -e . uvicorn
export TEST_ENVIRONMENT=true # set this when running locally to mock redis
uvicorn titiler.xarray.main:app --reload
uvicorn titiler.multidim.main:app --reload
```

To access the docs, visit http://127.0.0.1:8000/api.html.
To access the docs, visit <http://127.0.0.1:8000/api.html>.
![](https://github.com/developmentseed/titiler-xarray/assets/10407788/4368546b-5b60-4cd5-86be-fdd959374b17)

## Testing
Expand Down Expand Up @@ -46,7 +46,6 @@ The Github Actions workflow defined in [.github/workflows/ci.yml](./.github/work
* The production stack is deployed when the `main` branch is tagged, creating a new release. The production stack will deploy to a stack with an API Gateway associated with the domain prod-titiler-xarray.delta-backend.com/.
* The development stack will be deployed upon pushes to the `dev` and `main` branches. The development stack will deploy to a stack with an API Gateway associated with the domain dev-titiler-xarray.delta-backend.com/.


## New Deployments

The following steps detail how to to setup and deploy the CDK stack from your local machine.
Expand Down Expand Up @@ -94,7 +93,6 @@ The following steps detail how to to setup and deploy the CDK stack from your lo
AWS_DEFAULT_REGION=us-west-2 AWS_REGION=us-west-2 AWS_PROFILE=smce-veda STACK_STAGE=production npm --prefix infrastructure/aws run cdk -- deploy titiler-xarray-production
```


**Important**

In AWS Lambda environment we need to have specific version of botocore, S3FS, FSPEC and other libraries.
Expand Down
31 changes: 20 additions & 11 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "titiler.xarray"
description = "TiTiler extension for xarray."
name = "titiler-multidim"
description = "TiTiler application extension for titiler.xarray."
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
description = "TiTiler application extension for titiler.xarray."
description = "TiTiler application using titiler.xarray."

?

readme = "README.md"
requires-python = ">=3.8"
authors = [
Expand All @@ -25,6 +25,8 @@ classifiers = [
]
dynamic = ["version"]
dependencies = [
"titiler.core>=0.19.0,<0.20",
"titiler.xarray>=0.19.0,<0.20",
"cftime",
"h5netcdf",
"numpy<2.0.0",
Expand All @@ -36,13 +38,12 @@ dependencies = [
"s3fs",
"aiohttp",
"requests",
"pydantic==2.0.2",
"titiler.core>=0.14.1,<0.15",
"pydantic-settings~=2.0",
"pydantic>=2.4,<3.0",
"pandas==1.5.3",
"redis",
"fastapi>=0.100.0,<0.107.0",
"starlette<0.28",
"fastapi>=0.108.0,<0.109.0",
"starlette>=0.29.0,<0.33.0",
]

[project.optional-dependencies]
Expand All @@ -54,7 +55,10 @@ test = [
"yappi",
]
dev = [
"pre-commit"
"dask>=2023.5.0",
"ipython>=8.12.3",
"netcdf4>=1.7.2",
"pre-commit",
]
debug = [
"yappi"
Expand Down Expand Up @@ -111,13 +115,18 @@ namespace_packages = true
explicit_package_bases = true

[build-system]
requires = ["pdm-pep517"]
build-backend = "pdm.pep517.api"
requires = ["pdm-backend"]
build-backend = "pdm.backend"


[tool.pdm.version]
source = "file"
path = "titiler/xarray/__init__.py"
path = "src/titiler/multidim/__init__.py"


[tool.pdm]
package-dir = "src/"

[tool.pdm.build]
includes = ["titiler/xarray"]
includes = ["src/titiler/multidim/"]
excludes = ["tests/", "**/.mypy_cache", "**/.DS_Store"]
3 changes: 3 additions & 0 deletions src/titiler/multidim/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
"""titiler.multidim"""

__version__ = "0.3.0"
181 changes: 181 additions & 0 deletions src/titiler/multidim/factory.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
"""TiTiler.xarray factory."""

from typing import List, Literal, Optional, Type
from urllib.parse import urlencode

import jinja2
import numpy as np
from attrs import define
from fastapi import Depends, Query
from starlette.requests import Request
from starlette.responses import HTMLResponse
from starlette.templating import Jinja2Templates
from typing_extensions import Annotated

from titiler.core.dependencies import ColorFormulaParams, DefaultDependency
from titiler.core.resources.enums import ImageType
from titiler.core.resources.responses import JSONResponse
from titiler.multidim.reader import XarrayReader
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

non-blocking but now that I'm looking at this I am wondering if the naming convention should help distinguish local application dependencies vs imported dependencies... like titiler_app.multidim|xarray or titiler.app.multidim|xarrat

from titiler.xarray.dependencies import DatasetParams, XarrayIOParams, XarrayParams
from titiler.xarray.factory import TilerFactory as BaseTilerFactory


@define(kw_only=True)
class XarrayTilerFactory(BaseTilerFactory):
"""Xarray Tiler Factory."""

reader: Type[XarrayReader] = XarrayReader
reader_dependency: Type[DefaultDependency] = XarrayParams
dataset_dependency: Type[DefaultDependency] = DatasetParams

def register_routes(self) -> None: # noqa: C901
"""Register Info / Tiles / TileJSON endoints."""
super().register_routes()
self.variables()

def variables(self) -> None:
"""Register /variables endpoint"""

@self.router.get(
"/variables",
response_class=JSONResponse,
responses={200: {"description": "Return dataset's Variables."}},
)
def get_variables(
src_path=Depends(self.path_dependency),
io_params=Depends(XarrayIOParams),
) -> List[str]:
"""return available variables."""
return self.reader.list_variables(
src_path=src_path,
group=io_params.group,
decode_times=io_params.decode_times,
)

def statistics(self) -> None:
"""Register /statistics and /histogram endpoints"""
super().statistics()

@self.router.get(
"/histogram",
response_class=JSONResponse,
responses={200: {"description": "Return histogram for this data variable"}},
response_model_exclude_none=True,
)
def histogram(
src_path=Depends(self.path_dependency),
reader_params=Depends(self.reader_dependency),
):
with self.reader(
src_path=src_path,
variable=reader_params.variable,
group=reader_params.group,
decode_times=reader_params.decode_times,
datetime=reader_params.datetime,
) as src_dst:
boolean_mask = ~np.isnan(src_dst.input)
data_values = src_dst.input.values[boolean_mask]
counts, values = np.histogram(data_values, bins=10)
counts, values = counts.tolist(), values.tolist()
buckets = list(
zip(values, [values[i + 1] for i in range(len(values) - 1)])
)
hist_dict = []
for idx, bucket in enumerate(buckets):
hist_dict.append({"bucket": bucket, "value": counts[idx]})
return hist_dict

def map_viewer(self) -> None:
"""Register /map endpoints"""

@self.router.get("/{tileMatrixSetId}/map", response_class=HTMLResponse)
def map_viewer(
request: Request,
tileMatrixSetId: Annotated[ # type: ignore
Literal[tuple(self.supported_tms.list())],
"Identifier selecting one of the supported TileMatrixSetIds",
],
url: Annotated[Optional[str], Query(description="Dataset URL")] = None,
variable: Annotated[
Optional[str],
Query(description="Xarray Variable"),
] = None,
group: Annotated[
Optional[int],
Query(
description="Select a specific zarr group from a zarr hierarchy, can be for pyramids or datasets. Can be used to open a dataset in HDF5 files."
),
] = None,
decode_times: Annotated[
bool,
Query(
title="decode_times",
description="Whether to decode times",
),
] = True,
drop_dim: Annotated[
Optional[str],
Query(description="Dimension to drop"),
] = None,
datetime: Annotated[
Optional[str], Query(description="Slice of time to read (if available)")
] = None,
tile_format: Annotated[
Optional[ImageType],
Query(
description="Default will be automatically defined if the output image needs a mask (png) or not (jpeg).",
),
] = None,
tile_scale: Annotated[
int,
Query(
gt=0, lt=4, description="Tile size scale. 1=256x256, 2=512x512..."
),
] = 1,
minzoom: Annotated[
Optional[int],
Query(description="Overwrite default minzoom."),
] = None,
maxzoom: Annotated[
Optional[int],
Query(description="Overwrite default maxzoom."),
] = None,
post_process=Depends(self.process_dependency),
rescale=Depends(self.rescale_dependency),
color_formula=Depends(ColorFormulaParams),
colormap=Depends(self.colormap_dependency),
render_params=Depends(self.render_dependency),
dataset_params=Depends(self.dataset_dependency),
):
"""Return map Viewer."""
jinja2_env = jinja2.Environment(
loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, ".")])
)
templates = Jinja2Templates(env=jinja2_env)

if url:
tilejson_url = self.url_for(
request, "tilejson", tileMatrixSetId=tileMatrixSetId
)
if request.query_params._list:
tilejson_url += f"?{urlencode(request.query_params._list)}"

tms = self.supported_tms.get(tileMatrixSetId)
return templates.TemplateResponse(
name="map.html",
context={
"request": request,
"tilejson_endpoint": tilejson_url,
"tms": tms,
"resolutions": [matrix.cellSize for matrix in tms],
},
media_type="text/html",
)
else:
return templates.TemplateResponse(
name="map-form.html",
context={
"request": request,
},
media_type="text/html",
)
26 changes: 17 additions & 9 deletions titiler/xarray/main.py → src/titiler/multidim/main.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""titiler app."""
"""titiler.multidim."""

import logging

Expand All @@ -8,19 +8,19 @@
from starlette import status
from starlette.middleware.cors import CORSMiddleware

import titiler.xarray.reader as reader
import titiler.multidim.reader as reader
from titiler.core.errors import DEFAULT_STATUS_CODES, add_exception_handlers
from titiler.core.factory import AlgorithmFactory, TMSFactory
from titiler.core.factory import AlgorithmFactory, ColorMapFactory, TMSFactory
from titiler.core.middleware import (
CacheControlMiddleware,
LoggerMiddleware,
TotalTimeMiddleware,
)
from titiler.xarray import __version__ as titiler_version
from titiler.xarray.factory import ZarrTilerFactory
from titiler.xarray.middleware import ServerTimingMiddleware
from titiler.xarray.redis_pool import get_redis
from titiler.xarray.settings import ApiSettings
from titiler.multidim import __version__ as titiler_version
from titiler.multidim.factory import XarrayTilerFactory
from titiler.multidim.middleware import ServerTimingMiddleware
from titiler.multidim.redis_pool import get_redis
from titiler.multidim.settings import ApiSettings

logging.getLogger("botocore.credentials").disabled = True
logging.getLogger("botocore.utils").disabled = True
Expand All @@ -38,7 +38,7 @@

###############################################################################
# Tiles endpoints
xarray_factory = ZarrTilerFactory()
xarray_factory = XarrayTilerFactory()
app.include_router(xarray_factory.router, tags=["Xarray Tiler API"])

###############################################################################
Expand All @@ -51,6 +51,14 @@
algorithms = AlgorithmFactory()
app.include_router(algorithms.router, tags=["Algorithms"])

###############################################################################
# Colormaps endpoints
cmaps = ColorMapFactory()
app.include_router(
cmaps.router,
tags=["ColorMaps"],
)

error_codes = {
zarr.errors.GroupNotFoundError: status.HTTP_422_UNPROCESSABLE_ENTITY,
}
Expand Down
Loading