add legend information to coverages (#231)

* Adding bootstrap configs for collection and new historical variables

* Start adding config for historical TDd

* Added method to retrieve cov conf's main_netcdf_dataset_name

* Use cov_conf.get_main_netcdf_variable_name() when needing to retrieve the name of the netcdf main variable

* Added bootstrap conf param for observation-related year periods

* Added bootstrap conf param for historical datasets

* Added function to resolve thredds dataset names when a pattern is used to configure them

* Improved help text on admin fields

* Provide bootstrap configs for climatological datasets

* Enabled support for fnmatch patterns in dataset paths

* Add path operation for retrieving a single coverage identifier's details

* Ensure wms-related properties are correctly rendered

* Rename conf param value from 'collection' to 'archive'

* Rename conf param value from 'observation_variable' to 'historical_variable'

* Implement path operation to retrieve historical variable combinations

* Fixed failing tests

* Implement internal_value on conf param values

* Added a legend for coverage configurations

* Increased 95th percentile response time in load tests that run in CI

* Upped limit for load tests in CI

arpav_ppcv/config.py

@@ -93,6 +93,7 @@ class ArpavPpcvSettings(BaseSettings):  # noqa
     templates_dir: Optional[Path] = Path(__file__).parent / "webapp/templates"
     static_dir: Optional[Path] = Path(__file__).parent / "webapp/static"
     thredds_server: ThreddsServerSettings = ThreddsServerSettings()
+    palettes_dir: Path = Path(__file__).parents[1] / "data/palettes"
     prefect: PrefectSettings = PrefectSettings()
     martin_tile_server_base_url: str = "http://localhost:3000"
     nearest_station_radius_meters: int = 200

arpav_ppcv/palette.py

@@ -0,0 +1,40 @@
+import logging
+from pathlib import Path
+from typing import Optional
+
+logger = logging.getLogger(__name__)
+
+
+def parse_palette(palette: str, palettes_dir: Path) -> Optional[list[str]]:
+    palette_name = palette.partition("/")[-1].lower()
+    name, to_invert = palette_name.rpartition("-inv")[:2]
+    is_inverted = to_invert != ""
+    colors = []
+    for file_path in [f for f in palettes_dir.iterdir() if f.is_file()]:
+        if file_path.stem.lower() == name:
+            try:
+                colors = [
+                    line.strip()
+                    for line in file_path.read_text().splitlines()
+                    if line.startswith("#")
+                ]
+            except OSError:
+                logger.warning(f"Error reading file {file_path}")
+            break
+    else:
+        logger.warning(f"Could not find a palette named {name!r} at {palettes_dir!r}")
+    if is_inverted:
+        colors.reverse()
+    return colors if len(colors) > 0 else None
+
+
+def apply_palette(
+    colors: list[str], minimum: float, maximum: float
+) -> list[tuple[float, str]]:
+    minmax_range = maximum - minimum
+    step_increment = minmax_range / (len(colors) - 1)
+    result = []
+    for i, current_color in enumerate(colors):
+        current_value = minimum + i * step_increment
+        result.append((current_value, current_color))
+    return result

arpav_ppcv/thredds/crawler.py

@@ -3,7 +3,6 @@
 import logging
 import traceback
 import typing
-from itertools import islice
 from pathlib import Path
 from xml.etree import ElementTree as etree
 
@@ -12,8 +11,9 @@
 import exceptiongroup
 import httpx
 
-from ..schemas import coverages
 from .. import database
+from ..schemas import coverages
+from ..utils import batched
 
 logger = logging.getLogger(__name__)
 
@@ -154,7 +154,7 @@ async def download_datasets(
 ) -> None:
     client = httpx.AsyncClient()
     logger.debug(f"There are {len(dataset_urls)} URLS to process in total")
-    for batch in _batched(dataset_urls, 10):
+    for batch in batched(dataset_urls, 10):
         with exceptiongroup.catch({Exception: handle_thredds_download_exception}):
             async with anyio.create_task_group() as tg:
                 for dataset_url in batch:
@@ -201,20 +201,3 @@ async def download_individual_dataset(
                         fh.write(chunk)
     else:
         logger.info(f"dataset {output_path!r} already exists locally, skipping...")
-
-
-def _batched(iterable, n):
-    """Custom implementation of `itertools.batched()`.
-
-    This is a custom implementation of `itertools.batched()`, which is only available
-    on Python 3.12+. This is copied verbatim from the python docs at:
-
-    https://docs.python.org/3/library/itertools.html#itertools.batched
-
-    """
-    # batched('ABCDEFG', 3) --> ABC DEF G
-    if n < 1:
-        raise ValueError("n must be at least one")
-    it = iter(iterable)
-    while batch := tuple(islice(it, n)):
-        yield batch

arpav_ppcv/utils.py

@@ -0,0 +1,18 @@
+from itertools import islice
+
+
+def batched(iterable, n):
+    """Custom implementation of `itertools.batched()`.
+
+    This is a custom implementation of `itertools.batched()`, which is only available
+    on Python 3.12+. This is copied verbatim from the python docs at:
+
+    https://docs.python.org/3/library/itertools.html#itertools.batched
+
+    """
+    # batched('ABCDEFG', 3) --> ABC DEF G
+    if n < 1:
+        raise ValueError("n must be at least one")
+    it = iter(iterable)
+    while batch := tuple(islice(it, n)):
+        yield batch

arpav_ppcv/webapp/api_v2/routers/coverages.py

@@ -25,6 +25,7 @@
     database as db,
     exceptions,
     operations,
+    palette,
 )
 from ....config import ArpavPpcvSettings
 from ....thredds import (
@@ -171,6 +172,7 @@ def list_coverage_configurations(
 )
 def get_coverage_configuration(
     request: Request,
+    settings: Annotated[ArpavPpcvSettings, Depends(dependencies.get_settings)],
     db_session: Annotated[Session, Depends(dependencies.get_db_session)],
     coverage_configuration_id: pydantic.UUID4,
 ):
@@ -180,8 +182,19 @@ def get_coverage_configuration(
     allowed_coverage_identifiers = db.generate_coverage_identifiers(
         coverage_configuration=db_coverage_configuration
     )
+    palette_colors = palette.parse_palette(
+        db_coverage_configuration.palette, settings.palettes_dir
+    )
+    if palette_colors is not None:
+        applied_colors = palette.apply_palette(
+            palette_colors,
+            db_coverage_configuration.color_scale_min,
+            db_coverage_configuration.color_scale_max,
+        )
+    else:
+        applied_colors = []
     return coverage_schemas.CoverageConfigurationReadDetail.from_db_instance(
-        db_coverage_configuration, allowed_coverage_identifiers, request
+        db_coverage_configuration, allowed_coverage_identifiers, applied_colors, request
     )
 
 

arpav_ppcv/webapp/api_v2/schemas/coverages.py

@@ -12,6 +12,15 @@
 from .base import WebResourceList
 
 
+class ImageLegendColor(pydantic.BaseModel):
+    value: float
+    color: str
+
+
+class CoverageImageLegend(pydantic.BaseModel):
+    color_entries: list[ImageLegendColor]
+
+
 class ConfigurationParameterValueEmbeddedInConfigurationParameter(pydantic.BaseModel):
     name: str
     display_name_english: str
@@ -117,18 +126,17 @@ def from_db_instance(
 class CoverageConfigurationReadDetail(CoverageConfigurationReadListItem):
     url: pydantic.AnyHttpUrl
     unit: str
-    palette: str
-    color_scale_min: float
-    color_scale_max: float
     allowed_coverage_identifiers: list[str]
     description_english: str | None
     description_italian: str | None
+    legend: CoverageImageLegend
 
     @classmethod
     def from_db_instance(
         cls,
         instance: app_models.CoverageConfiguration,
         allowed_coverage_identifiers: list[str],
+        legend_colors: list[tuple[float, str]],
         request: Request,
     ) -> "CoverageConfigurationReadDetail":
         url = request.url_for(
@@ -153,6 +161,11 @@ def from_db_instance(
                 for pv in instance.possible_values
             ],
             allowed_coverage_identifiers=allowed_coverage_identifiers,
+            legend=CoverageImageLegend(
+                color_entries=[
+                    ImageLegendColor(value=v, color=c) for v, c in legend_colors
+                ]
+            ),
         )
 
 

data/palettes/README.md

@@ -0,0 +1,10 @@
+# Palettes
+
+This directory contains a copy of the palette files used by ncWMS. These were obtained by copying them from
+the edal-java source code repository, available at:
+
+https://github.com/Reading-eScience-Centre/edal-java
+
+In the edal-java repo, these palette files are found under:
+
+`graphics/src/main/resources/palettes`

data/palettes/colorbrewer_licence.txt

@@ -0,0 +1,22 @@
+Apache-Style Software License for ColorBrewer software and ColorBrewer Color Schemes
+
+Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania State University.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and limitations under the License.
+
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+1. Redistributions as source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. The end-user documentation included with the redistribution, if any, must include the following acknowledgment:
+This product includes color specifications and designs developed by Cynthia Brewer (http://colorbrewer.org/).
+Alternately, this acknowledgment may appear in the software itself, if and wherever such third-party acknowledgments normally appear.
+4. The name "ColorBrewer" must not be used to endorse or promote products derived from this software without prior written permission.
+For written permission, please contact Cynthia Brewer at [email protected].
+5. Products derived from this software may not be called "ColorBrewer", nor may "ColorBrewer" appear in their name, without prior written permission of Cynthia Brewer.

data/palettes/div-BrBG.pal

@@ -0,0 +1,11 @@
+#FF543005
+#FF8c510a
+#FFbf812d
+#FFdfc27d
+#FFf6e8c3
+#FFf5f5f5
+#FFc7eae5
+#FF80cdc1
+#FF35978f
+#FF01665e
+#FF003c30

data/palettes/div-BuRd.pal

@@ -0,0 +1,4 @@
+% redblue
+#FF0000FF
+#FFFFFFFF
+#FFFF0000

data/palettes/div-BuRd2.pal

@@ -0,0 +1,11 @@
+#FF053061
+#FF2166ac
+#FF4393c3
+#FF92c5de
+#FFd1e5f0
+#FFf7f7f7
+#FFfddbc7
+#FFf4a582
+#FFd6604d
+#FFb2182b
+#FF67001f

data/palettes/div-PRGn.pal

@@ -0,0 +1,11 @@
+#FF40004b
+#FF762a83
+#FF9970ab
+#FFc2a5cf
+#FFe7d4e8
+#FFf7f7f7
+#FFd9f0d3
+#FFa6dba0
+#FF5aae61
+#FF1b7837
+#FF00441b

data/palettes/div-PiYG.pal

@@ -0,0 +1,11 @@
+#FF8e0152
+#FFc51b7d
+#FFde77ae
+#FFf1b6da
+#FFfde0ef
+#FFf7f7f7
+#FFe6f5d0
+#FFb8e186
+#FF7fbc41
+#FF4d9221
+#FF276419

data/palettes/div-PuOr.pal

@@ -0,0 +1,11 @@
+#FF7f3b08
+#FFb35806
+#FFe08214
+#FFfdb863
+#FFfee0b6
+#FFf7f7f7
+#FFd8daeb
+#FFb2abd2
+#FF8073ac
+#FF542788
+#FF2d004b

data/palettes/div-RdBu.pal

@@ -0,0 +1,11 @@
+#FF67001f
+#FFb2182b
+#FFd6604d
+#FFf4a582
+#FFfddbc7
+#FFf7f7f7
+#FFd1e5f0
+#FF92c5de
+#FF4393c3
+#FF2166ac
+#FF053061

data/palettes/div-RdGy.pal

@@ -0,0 +1,11 @@
+#FF67001f
+#FFb2182b
+#FFd6604d
+#FFf4a582
+#FFfddbc7
+#FFffffff
+#FFe0e0e0
+#FFbababa
+#FF878787
+#FF4d4d4d
+#FF1a1a1a

data/palettes/div-RdYlBu.pal

@@ -0,0 +1,11 @@
+#FFa50026
+#FFd73027
+#FFf46d43
+#FFfdae61
+#FFfee090
+#FFffffbf
+#FFe0f3f8
+#FFabd9e9
+#FF74add1
+#FF4575b4
+#FF313695

data/palettes/div-RdYlGn.pal

@@ -0,0 +1,11 @@
+#FFa50026
+#FFd73027
+#FFf46d43
+#FFfdae61
+#FFfee08b
+#FFffffbf
+#FFd9ef8b
+#FFa6d96a
+#FF66bd63
+#FF1a9850
+#FF006837

data/palettes/div-Spectral.pal

@@ -0,0 +1,11 @@
+#FF9e0142
+#FFd53e4f
+#FFf46d43
+#FFfdae61
+#FFfee08b
+#FFffffbf
+#FFe6f598
+#FFabdda4
+#FF66c2a5
+#FF3288bd
+#FF5e4fa2