From 8b05a7f13c2f11c138fbdf0e6b497a4dca5a53cb Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Fri, 29 Nov 2024 15:05:43 +0000 Subject: [PATCH 1/4] Generate WMS legend values dynamically and expose API path for it --- arpav_ppcv/operations.py | 53 +++++++++++++++++++ arpav_ppcv/thredds/utils.py | 4 ++ arpav_ppcv/webapp/api_v2/routers/coverages.py | 29 ++++++++++ 3 files changed, 86 insertions(+) diff --git a/arpav_ppcv/operations.py b/arpav_ppcv/operations.py index c7dfd360..72bafe00 100644 --- a/arpav_ppcv/operations.py +++ b/arpav_ppcv/operations.py @@ -33,6 +33,7 @@ from . import ( config, database, + palette, ) from .schemas import ( base, @@ -104,6 +105,58 @@ def get_climate_barometer_time_series( return result +def apply_palette_to_coverage( + settings: config.ArpavPpcvSettings, + coverage: coverages.CoverageInternal, + temporal_instant: Optional[dt.datetime] = None, +) -> list[tuple[float, str]]: + opendap_url = "/".join( + ( + settings.thredds_server.base_url, + settings.thredds_server.opendap_service_url_fragment, + crawler.get_thredds_url_fragment( + coverage, settings.thredds_server.base_url + ), + ) + ) + if temporal_instant is not None: + ds = netCDF4.Dataset(opendap_url) + netcdf_variable_name = coverage.configuration.get_main_netcdf_variable_name( + coverage.identifier + ) + time_var = ds["time"] + time_index = cftime.date2index( + temporal_instant, time_var, time_var.calendar, select="nearest" + ) + found_instant = cftime.num2pydate( + time_index, units=time_var.units, calendar=time_var.calendar + ) + logger.info(f"Found temporal instant {found_instant}") + data_max = np.nanmax(ds[netcdf_variable_name][time_index, :, :]) + data_min = np.nanmin(ds[netcdf_variable_name][time_index, :, :]) + else: + data_max = coverage.configuration.color_scale_max + data_min = coverage.configuration.color_scale_min + palette_colors = palette.parse_palette( + coverage.configuration.palette, settings.palettes_dir + ) + applied_colors = [] + if palette_colors is not None: + if abs(data_max - data_min) > 0.001: + applied_colors = palette.apply_palette( + palette_colors, data_min, data_max, num_stops=settings.palette_num_stops + ) + else: + logger.warning( + f"Cannot calculate applied colors for coverage " + f"configuration {coverage.configuration.name!r} - check the " + f"colorscale min and max values" + ) + else: + logger.warning(f"Unable to parse palette {coverage.configuration.palette!r}") + return applied_colors + + def _get_climate_barometer_data( settings: config.ArpavPpcvSettings, coverage: coverages.CoverageInternal, diff --git a/arpav_ppcv/thredds/utils.py b/arpav_ppcv/thredds/utils.py index 6fa72aa2..078c512b 100644 --- a/arpav_ppcv/thredds/utils.py +++ b/arpav_ppcv/thredds/utils.py @@ -58,6 +58,7 @@ def tweak_wms_get_map_request( num_color_bands = "2" query_params["NUMCOLORBANDS"] = num_color_bands else: + num_color_bands = "250" if "uncertainty_group" in layer_name: palette = ncwms_palette else: @@ -75,4 +76,7 @@ def tweak_wms_get_map_request( query_params["styles"] = palette query_params["colorscalerange"] = color_scale_range + query_params["NUMCOLORBANDS"] = num_color_bands + query_params["ABOVEMAXCOLOR"] = "extend" + query_params["BELOWMINCOLOR"] = "extend" return query_params diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index cce31dc5..19ac4b9f 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -1,3 +1,4 @@ +import datetime as dt import logging import urllib.parse from operator import itemgetter @@ -285,6 +286,34 @@ def get_coverage_identifier( raise HTTPException(400, detail=_INVALID_COVERAGE_IDENTIFIER_ERROR_DETAIL) +@router.get( + "/wms-legend/{coverage_identifier}", + response_model=coverage_schemas.CoverageImageLegend, +) +def get_wms_legend( + request: Request, + db_session: Annotated[Session, Depends(dependencies.get_db_session)], + settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], + coverage_identifier: str, + datetime: Optional[dt.datetime] = None, +): + """Get legend for WMS GetMap calls""" + if (coverage := db.get_coverage(db_session, coverage_identifier)) is not None: + applied_colors = operations.apply_palette_to_coverage( + settings, coverage, datetime + ) + return coverage_schemas.CoverageImageLegend( + color_entries=[ + coverage_schemas.ImageLegendColor(value=v, color=c) + for v, c in applied_colors + ] + ) + else: + raise HTTPException( + status_code=400, detail=_INVALID_COVERAGE_IDENTIFIER_ERROR_DETAIL + ) + + @router.get("/wms/{coverage_identifier}") async def wms_endpoint( request: Request, From aeff2835a19d141a178d819477d3374651029f25 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Mon, 2 Dec 2024 13:11:06 +0000 Subject: [PATCH 2/4] Not providing static WMS legend information anymore We now have the dynamic legend endpoint, which should be used instead --- arpav_ppcv/webapp/api_v2/routers/coverages.py | 26 +------------------ arpav_ppcv/webapp/api_v2/schemas/coverages.py | 7 ----- 2 files changed, 1 insertion(+), 32 deletions(-) diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index 19ac4b9f..223b78c0 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -30,7 +30,6 @@ datadownloads, exceptions, operations, - palette, ) from ....config import ArpavPpcvSettings from ....thredds import ( @@ -178,7 +177,6 @@ def list_coverage_configurations( ) def get_coverage_configuration( request: Request, - settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], db_session: Annotated[Session, Depends(dependencies.get_db_session)], coverage_configuration_id: pydantic.UUID4, ): @@ -188,33 +186,11 @@ def get_coverage_configuration( allowed_coverage_identifiers = db.generate_coverage_identifiers( coverage_configuration=db_coverage_configuration ) - palette_colors = palette.parse_palette( - db_coverage_configuration.palette, settings.palettes_dir - ) - applied_colors = [] - if palette_colors is not None: - minimum = db_coverage_configuration.color_scale_min - maximum = db_coverage_configuration.color_scale_max - if abs(maximum - minimum) > 0.001: - applied_colors = palette.apply_palette( - palette_colors, minimum, maximum, num_stops=settings.palette_num_stops - ) - else: - logger.warning( - f"Cannot calculate applied colors for coverage " - f"configuration {db_coverage_configuration.name!r} - check the " - f"colorscale min and max values" - ) - else: - logger.warning(f"Unable to parse palette {db_coverage_configuration.palette!r}") return coverage_schemas.CoverageConfigurationReadDetail.from_db_instance( - db_coverage_configuration, allowed_coverage_identifiers, applied_colors, request + db_coverage_configuration, allowed_coverage_identifiers, request ) -# PossibleValue: pydantic.StringConstraints(pattern="^[\w-_]+:[\w-_]+$") - - @router.get( "/coverage-identifiers", response_model=coverage_schemas.CoverageIdentifierList, diff --git a/arpav_ppcv/webapp/api_v2/schemas/coverages.py b/arpav_ppcv/webapp/api_v2/schemas/coverages.py index c6784d0d..8687f520 100644 --- a/arpav_ppcv/webapp/api_v2/schemas/coverages.py +++ b/arpav_ppcv/webapp/api_v2/schemas/coverages.py @@ -135,7 +135,6 @@ class CoverageConfigurationReadDetail(CoverageConfigurationReadListItem): allowed_coverage_identifiers: list[str] description_english: str | None description_italian: str | None - legend: CoverageImageLegend data_precision: int @classmethod @@ -143,7 +142,6 @@ def from_db_instance( cls, instance: app_models.CoverageConfiguration, allowed_coverage_identifiers: list[str], - legend_colors: list[tuple[float, str]], request: Request, ) -> "CoverageConfigurationReadDetail": url = request.url_for( @@ -168,11 +166,6 @@ def from_db_instance( for pv in instance.possible_values ], allowed_coverage_identifiers=allowed_coverage_identifiers, - legend=CoverageImageLegend( - color_entries=[ - ImageLegendColor(value=v, color=c) for v, c in legend_colors - ] - ), ) From 2b930aa0f2cbb90b1086a0208b58ba62413fbec1 Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 5 Dec 2024 12:51:07 +0000 Subject: [PATCH 3/4] Reverted previous implementation of dynamic WMS legend generation This is not needed, as the underlying THREDDS WMS server does not perform dynamic contrast enhancement --- arpav_ppcv/operations.py | 53 ------------------ arpav_ppcv/webapp/api_v2/routers/coverages.py | 55 +++++++++---------- arpav_ppcv/webapp/api_v2/schemas/coverages.py | 7 +++ 3 files changed, 32 insertions(+), 83 deletions(-) diff --git a/arpav_ppcv/operations.py b/arpav_ppcv/operations.py index 72bafe00..c7dfd360 100644 --- a/arpav_ppcv/operations.py +++ b/arpav_ppcv/operations.py @@ -33,7 +33,6 @@ from . import ( config, database, - palette, ) from .schemas import ( base, @@ -105,58 +104,6 @@ def get_climate_barometer_time_series( return result -def apply_palette_to_coverage( - settings: config.ArpavPpcvSettings, - coverage: coverages.CoverageInternal, - temporal_instant: Optional[dt.datetime] = None, -) -> list[tuple[float, str]]: - opendap_url = "/".join( - ( - settings.thredds_server.base_url, - settings.thredds_server.opendap_service_url_fragment, - crawler.get_thredds_url_fragment( - coverage, settings.thredds_server.base_url - ), - ) - ) - if temporal_instant is not None: - ds = netCDF4.Dataset(opendap_url) - netcdf_variable_name = coverage.configuration.get_main_netcdf_variable_name( - coverage.identifier - ) - time_var = ds["time"] - time_index = cftime.date2index( - temporal_instant, time_var, time_var.calendar, select="nearest" - ) - found_instant = cftime.num2pydate( - time_index, units=time_var.units, calendar=time_var.calendar - ) - logger.info(f"Found temporal instant {found_instant}") - data_max = np.nanmax(ds[netcdf_variable_name][time_index, :, :]) - data_min = np.nanmin(ds[netcdf_variable_name][time_index, :, :]) - else: - data_max = coverage.configuration.color_scale_max - data_min = coverage.configuration.color_scale_min - palette_colors = palette.parse_palette( - coverage.configuration.palette, settings.palettes_dir - ) - applied_colors = [] - if palette_colors is not None: - if abs(data_max - data_min) > 0.001: - applied_colors = palette.apply_palette( - palette_colors, data_min, data_max, num_stops=settings.palette_num_stops - ) - else: - logger.warning( - f"Cannot calculate applied colors for coverage " - f"configuration {coverage.configuration.name!r} - check the " - f"colorscale min and max values" - ) - else: - logger.warning(f"Unable to parse palette {coverage.configuration.palette!r}") - return applied_colors - - def _get_climate_barometer_data( settings: config.ArpavPpcvSettings, coverage: coverages.CoverageInternal, diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index 223b78c0..cce31dc5 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -1,4 +1,3 @@ -import datetime as dt import logging import urllib.parse from operator import itemgetter @@ -30,6 +29,7 @@ datadownloads, exceptions, operations, + palette, ) from ....config import ArpavPpcvSettings from ....thredds import ( @@ -177,6 +177,7 @@ def list_coverage_configurations( ) def get_coverage_configuration( request: Request, + settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], db_session: Annotated[Session, Depends(dependencies.get_db_session)], coverage_configuration_id: pydantic.UUID4, ): @@ -186,11 +187,33 @@ def get_coverage_configuration( allowed_coverage_identifiers = db.generate_coverage_identifiers( coverage_configuration=db_coverage_configuration ) + palette_colors = palette.parse_palette( + db_coverage_configuration.palette, settings.palettes_dir + ) + applied_colors = [] + if palette_colors is not None: + minimum = db_coverage_configuration.color_scale_min + maximum = db_coverage_configuration.color_scale_max + if abs(maximum - minimum) > 0.001: + applied_colors = palette.apply_palette( + palette_colors, minimum, maximum, num_stops=settings.palette_num_stops + ) + else: + logger.warning( + f"Cannot calculate applied colors for coverage " + f"configuration {db_coverage_configuration.name!r} - check the " + f"colorscale min and max values" + ) + else: + logger.warning(f"Unable to parse palette {db_coverage_configuration.palette!r}") return coverage_schemas.CoverageConfigurationReadDetail.from_db_instance( - db_coverage_configuration, allowed_coverage_identifiers, request + db_coverage_configuration, allowed_coverage_identifiers, applied_colors, request ) +# PossibleValue: pydantic.StringConstraints(pattern="^[\w-_]+:[\w-_]+$") + + @router.get( "/coverage-identifiers", response_model=coverage_schemas.CoverageIdentifierList, @@ -262,34 +285,6 @@ def get_coverage_identifier( raise HTTPException(400, detail=_INVALID_COVERAGE_IDENTIFIER_ERROR_DETAIL) -@router.get( - "/wms-legend/{coverage_identifier}", - response_model=coverage_schemas.CoverageImageLegend, -) -def get_wms_legend( - request: Request, - db_session: Annotated[Session, Depends(dependencies.get_db_session)], - settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)], - coverage_identifier: str, - datetime: Optional[dt.datetime] = None, -): - """Get legend for WMS GetMap calls""" - if (coverage := db.get_coverage(db_session, coverage_identifier)) is not None: - applied_colors = operations.apply_palette_to_coverage( - settings, coverage, datetime - ) - return coverage_schemas.CoverageImageLegend( - color_entries=[ - coverage_schemas.ImageLegendColor(value=v, color=c) - for v, c in applied_colors - ] - ) - else: - raise HTTPException( - status_code=400, detail=_INVALID_COVERAGE_IDENTIFIER_ERROR_DETAIL - ) - - @router.get("/wms/{coverage_identifier}") async def wms_endpoint( request: Request, diff --git a/arpav_ppcv/webapp/api_v2/schemas/coverages.py b/arpav_ppcv/webapp/api_v2/schemas/coverages.py index 8687f520..c6784d0d 100644 --- a/arpav_ppcv/webapp/api_v2/schemas/coverages.py +++ b/arpav_ppcv/webapp/api_v2/schemas/coverages.py @@ -135,6 +135,7 @@ class CoverageConfigurationReadDetail(CoverageConfigurationReadListItem): allowed_coverage_identifiers: list[str] description_english: str | None description_italian: str | None + legend: CoverageImageLegend data_precision: int @classmethod @@ -142,6 +143,7 @@ def from_db_instance( cls, instance: app_models.CoverageConfiguration, allowed_coverage_identifiers: list[str], + legend_colors: list[tuple[float, str]], request: Request, ) -> "CoverageConfigurationReadDetail": url = request.url_for( @@ -166,6 +168,11 @@ def from_db_instance( for pv in instance.possible_values ], allowed_coverage_identifiers=allowed_coverage_identifiers, + legend=CoverageImageLegend( + color_entries=[ + ImageLegendColor(value=v, color=c) for v, c in legend_colors + ] + ), ) From 6f59570539facd99eb419d82164ec3def3fd06bb Mon Sep 17 00:00:00 2001 From: Ricardo Garcia Silva Date: Thu, 5 Dec 2024 13:00:36 +0000 Subject: [PATCH 4/4] Set a default timeout of 30s to http client --- arpav_ppcv/config.py | 2 +- arpav_ppcv/webapp/api_v2/routers/coverages.py | 3 --- arpav_ppcv/webapp/dependencies.py | 12 ++++++++---- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/arpav_ppcv/config.py b/arpav_ppcv/config.py index 6a169756..5864df82 100644 --- a/arpav_ppcv/config.py +++ b/arpav_ppcv/config.py @@ -47,7 +47,6 @@ class ThreddsServerSettings(pydantic.BaseModel): base_url: str = "http://localhost:8080/thredds" wms_service_url_fragment: str = "wms" netcdf_subset_service_url_fragment: str = "ncss/grid" # noqa - netcdf_subset_service_timeout_seconds: int = 30 # noqa opendap_service_url_fragment: str = "dodsC" # noqa uncertainty_visualization_scale_range: tuple[float, float] = pydantic.Field( default=(0, 9) @@ -132,6 +131,7 @@ class ArpavPpcvSettings(BaseSettings): # noqa coverage_download_settings: CoverageDownloadSettings = CoverageDownloadSettings() variable_stations_db_schema: str = "stations" num_uvicorn_worker_processes: int = 1 + http_client_timeout_seconds: float = 30.0 @pydantic.model_validator(mode="after") def ensure_test_db_dsn(self): diff --git a/arpav_ppcv/webapp/api_v2/routers/coverages.py b/arpav_ppcv/webapp/api_v2/routers/coverages.py index cce31dc5..673e71b2 100644 --- a/arpav_ppcv/webapp/api_v2/routers/coverages.py +++ b/arpav_ppcv/webapp/api_v2/routers/coverages.py @@ -452,9 +452,6 @@ async def get_forecast_data( fitted_bbox = None cache_key = datadownloads.get_cache_key(coverage, fitted_bbox, temporal_range) - http_client.timeout = ( - settings.thredds_server.netcdf_subset_service_timeout_seconds - ) response_to_stream = await datadownloads.retrieve_coverage_data( settings, http_client, cache_key, coverage, fitted_bbox, temporal_range ) diff --git a/arpav_ppcv/webapp/dependencies.py b/arpav_ppcv/webapp/dependencies.py index 161cf330..209baf84 100644 --- a/arpav_ppcv/webapp/dependencies.py +++ b/arpav_ppcv/webapp/dependencies.py @@ -29,12 +29,16 @@ def get_db_session(engine=Depends(get_db_engine)): # noqa: B008 yield session -def get_http_client() -> httpx.AsyncClient: - return httpx.AsyncClient() +def get_http_client( + settings: config.ArpavPpcvSettings = Depends(get_settings), +) -> httpx.AsyncClient: + return httpx.AsyncClient(timeout=settings.http_client_timeout_seconds) -def get_sync_http_client() -> httpx.Client: - return httpx.Client() +def get_sync_http_client( + settings: config.ArpavPpcvSettings = Depends(get_settings), +) -> httpx.Client: + return httpx.Client(timeout=settings.http_client_timeout_seconds) class CommonListFilterParameters(pydantic.BaseModel): # noqa: D101