diff --git a/Dockerfile b/Dockerfile
index 5ef77c10..d28621fd 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,7 +6,7 @@ USER root
LABEL maintainer="adaguc@knmi.nl"
# Version should be same as in Definitions.h
-LABEL version="2.21.2"
+LABEL version="2.22.0"
# Try to update image packages
RUN apt-get -q -y update \
diff --git a/NEWS.md b/NEWS.md
index 7ebb35d3..babb84e5 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,3 +1,8 @@
+**Version 2.22.0 2024-05-22**
+- EDR: Parameters can now be detailed with metadata like standard_names and units: https://github.com/KNMI/adaguc-server/issues/359.
+- See [Configure_EDR_service](doc/tutorials/Configure_EDR_service.md) for details.
+
+
**Version 2.21.2 2024-04-26**
- When using docker compose the Redis container now automatically starts when system restarts
diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h
index e218206c..14d8ca80 100755
--- a/adagucserverEC/Definitions.h
+++ b/adagucserverEC/Definitions.h
@@ -28,7 +28,7 @@
#ifndef Definitions_H
#define Definitions_H
-#define ADAGUCSERVER_VERSION "2.21.2" // Please also update in the Dockerfile to the same version
+#define ADAGUCSERVER_VERSION "2.22.0" // Please also update in the Dockerfile to the same version
// CConfigReaderLayerType
#define CConfigReaderLayerTypeUnknown 0
diff --git a/data/config/datasets/netcdf_5d.xml b/data/config/datasets/netcdf_5d.xml
index fec34846..062fcb27 100644
--- a/data/config/datasets/netcdf_5d.xml
+++ b/data/config/datasets/netcdf_5d.xml
@@ -5,6 +5,13 @@
+
+
@@ -38,6 +45,14 @@
+
+ data_extra_metadata
+ data_extra_metadata
+ {ADAGUC_PATH}/data/datasets/netcdf_5dims
+ data
+ testdata
+
+
diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md
index a396a24d..5326bf76 100644
--- a/doc/tutorials/Configure_EDR_service.md
+++ b/doc/tutorials/Configure_EDR_service.md
@@ -23,14 +23,19 @@ This file is available in the adaguc-server repository with location `data/datas
Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can also consider changing `` to `/data/adaguc-data/*.nc`.
```xml
-
+
-
+
-
+
@@ -43,18 +48,63 @@ Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can
+
- /data/adaguc-data/HARM_N25_20171215090000_dimx16_dimy16_dimtime49_dimforecastreferencetime1_varairtemperatureat2m.nc
+
+
+ /data/adaguc-data/HARM_N25_20171215090000_dimx16_dimy16_dimtime49_dimforecastreferencetime1_varairtemperatureat2m.nc
air_temperature__at_2m
temperature
-
+```
+
+
+### EdrParameter settings
+```xml
+
+```
+
+- name: Mandatory, Should be one of the WMS Layer names as advertised in the WMS GetCapabilities
+- unit: Mandatory, Sets the unit for the parameter in the parameter_names section of the collection document
+- standard_name: Recommended, sets the observedProperty id. If set the id will contain a link to the vocabulary service. If not set, it will fallback to `name` and `observedProperty.id` will not contain a link.
+- observed_property_label: Recommended, sets the observedProperty label, it will fallback to it will fallback first to `standard_name` first, and second to `name`
+- parameter_label: Recommended, sets the label for the parameter in the parameter_names section, it will fallback to the `name`
+
+For the given example this will result in the following parameter name definition:
+
+```json
+parameter_names": {
+ "air_temperature__at_2m": {
+ "type": "Parameter",
+ "id": "air_temperature__at_2m",
+ "label": "Air temperature, 2 metre",
+ "description": "harmonie - air_temperature__at_2m (air_temperature__at_2m)",
+ "unit": {
+ "symbol": {
+ "value": "°C",
+ "type": "http://www.opengis.net/def/uom/UCUM"
+ }
+ },
+ "observedProperty": {
+ "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
+ "label": "Air temperature"
+ }
+ }
+ }
```
+*The description is currently read from the GetCapabilities document, using the WMS Layer Title section prefixed with the edr collection name.
+
+
## Step 3: Scan the new data
```
diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py
index 96835c1b..f2bc5d47 100644
--- a/python/python_fastapi_server/routers/edr.py
+++ b/python/python_fastapi_server/routers/edr.py
@@ -29,6 +29,7 @@
ReferenceSystemConnectionObject,
)
from covjson_pydantic.unit import Unit as CovJsonUnit
+from covjson_pydantic.unit import Symbol as CovJsonSymbol
from defusedxml.ElementTree import ParseError, parse
from edr_pydantic.capabilities import (
@@ -44,6 +45,8 @@
from edr_pydantic.observed_property import ObservedProperty
from edr_pydantic.parameter import Parameter
from edr_pydantic.unit import Unit
+from edr_pydantic.unit import Symbol
+
from edr_pydantic.variables import Variables
from fastapi import FastAPI, Query, Request, Response
@@ -65,6 +68,9 @@
OWSLIB_DUMMY_URL = "http://localhost:8000"
+SYMBOL_TYPE_URL = "http://www.opengis.net/def/uom/UCUM"
+VOCAB_ENDPOINT_URL = "https://vocab.nerc.ac.uk/standard_name/"
+
def get_base_url(req: Request = None) -> str:
"""Returns the base url of this service"""
@@ -126,12 +132,41 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI
"name" in edr_parameter.attrib
and "unit" in edr_parameter.attrib
):
- edr_params.append(
- {
- "name": edr_parameter.attrib.get("name"),
- "unit": edr_parameter.attrib.get("unit"),
- }
- )
+ name = edr_parameter.attrib.get("name")
+ edr_param = {
+ "name": name,
+ "parameter_label":name,
+ "observed_property_label":name,
+ "description":name,
+ "standard_name":None,
+ "unit": "-",
+ }
+ # Try to take the standard name from the configuration
+ if "standard_name" in edr_parameter.attrib:
+ edr_param["standard_name"] = (
+ edr_parameter.attrib.get("standard_name")
+ )
+ # If there is a standard name, set the label to the same as fallback
+ edr_param["observed_property_label"] = (
+ edr_parameter.attrib.get("standard_name")
+ )
+
+ # Try to take the observed_property_label from the configuration
+ if "observed_property_label" in edr_parameter.attrib:
+ edr_param["observed_property_label"] = edr_parameter.attrib.get(
+ "observed_property_label"
+ )
+ # Try to take the parameter_label from the Layer configuration Title
+ if "parameter_label" in edr_parameter.attrib:
+ edr_param["parameter_label"] = edr_parameter.attrib.get(
+ "parameter_label")
+ # Try to take the parameter_label from the Layer configuration Title
+ if "unit" in edr_parameter.attrib:
+ edr_param["unit"] = edr_parameter.attrib.get(
+ "unit")
+
+ edr_params.append(edr_param)
+
else:
logger.warning(
"In dataset %s, skipping parameter %s: has no name or units configured",
@@ -165,7 +200,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI
# The edr_collections information is cached locally for a maximum of 2 minutes
# It will be refreshed if older than 2 minutes
-edr_cache = TTLCache(maxsize=100, ttl=120)
+edr_cache = TTLCache(maxsize=100, ttl=120) # TODO: Redis?
@cached(cache=edr_cache)
@@ -286,7 +321,7 @@ async def get_coll_inst_position(
if ttl is not None:
response.headers["cache-control"] = generate_max_age(ttl)
return covjson_from_resp(
- dat, edr_collections[collection_name]["vertical_name"]
+ dat, edr_collections[collection_name]["vertical_name"], collection_name
)
raise EdrException(code=400, description="No data")
@@ -309,7 +344,7 @@ async def get_collectioninfo_for_id(
logger.info("get_collectioninfo_for_id(%s, %s)", edr_collection, instance)
edr_collectionsinfo = get_edr_collections()
if edr_collection not in edr_collectionsinfo:
- raise EdrException(code=400, description="No data")
+ raise EdrException(code=400, description="Unknown or unconfigured collection")
edr_collectioninfo = edr_collectionsinfo[edr_collection]
@@ -412,7 +447,7 @@ async def get_collectioninfo_for_id(
else:
data_queries = DataQueries(position=EDRQuery(link=position_link))
- parameter_names = get_params_for_collection(edr_collection=edr_collection)
+ parameter_names = get_params_for_collection(edr_collection=edr_collection, wmslayers=wmslayers)
crs = ["EPSG:4326"]
@@ -440,28 +475,71 @@ async def get_collectioninfo_for_id(
return collection, ttl
+def get_parameter_config(edr_collection_name:str, param_id:str):
+ """Gets the EDRParameter configuration based on the edr collection name and parameter id
+
+ Args:
+ edr_collection_name (str): edr collection name
+ param_id (str): parameter id
+
+ Returns:
+ _type_: parameter element_
+ """
+ edr_collections = get_edr_collections()
+ edr_collection_parameters = edr_collections[edr_collection_name]["parameters"]
+ for param_el in edr_collection_parameters:
+ if param_id == param_el["name"]:
+ return param_el
+ return None
+
+def get_param_metadata(param_id:str, edr_collection_name)->dict:
+ """Composes parameter metadata based on the param_el and the wmslayer dictionaries
+
+ Args:
+ param_id (str): The parameter / wms layer name to find
+ edr_collection_name (str): The collection name
-def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]:
+ Returns:
+ dict: dictionary with all metadata required to construct a Edr Parameter object.
+ """
+ param_el = get_parameter_config(edr_collection_name, param_id)
+ wms_layer_name = param_el["name"]
+ observed_property_id = wms_layer_name
+ parameter_label = param_el["parameter_label"]
+ parameter_unit = param_el["unit"]
+ observed_property_label = param_el["observed_property_label"]
+ if "standard_name" in param_el and param_el["standard_name"] is not None:
+ observed_property_id = VOCAB_ENDPOINT_URL + param_el["standard_name"]
+
+ return { "wms_layer_name":wms_layer_name,
+ "observed_property_id":observed_property_id,
+ "observed_property_label":observed_property_label,
+ "parameter_label":parameter_label,
+ "parameter_unit":parameter_unit}
+
+def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, Parameter]:
"""
Returns a dictionary with parameters for given EDR collection
"""
parameter_names = {}
edr_collections = get_edr_collections()
for param_el in edr_collections[edr_collection]["parameters"]:
- # Use name as default for label
- if "label" in param_el:
- label = param_el["label"]
+ param_id = param_el["name"]
+ if not param_id in wmslayers:
+ logger.warning("EDR Parameter with name [%s] is not found in any of the adaguc Layer configurations. Available layers are %s", param_id, str(list(wmslayers.keys())))
else:
- label = param_el["name"]
-
- param = Parameter(
- id=param_el["name"],
- observedProperty=ObservedProperty(id=param_el["name"], label=label),
- type="Parameter",
- unit=Unit(symbol=param_el["unit"]),
- label=label,
- )
- parameter_names[param_el["name"]] = param
+ param_metadata = get_param_metadata(param_id, edr_collection)
+ logger.info(param_id)
+ logger.info(param_metadata["wms_layer_name"])
+ param = Parameter(
+ id=param_metadata["wms_layer_name"],
+ observedProperty=ObservedProperty(id=param_metadata["observed_property_id"], label=param_metadata["observed_property_label"]),
+ # description=param_metadata["wms_layer_title"], # TODO in follow up
+ type="Parameter",
+ unit=Unit(symbol=Symbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL)),
+ label=param_metadata["parameter_label"]
+ )
+ parameter_names[param_el["name"]] = param
return parameter_names
@@ -728,6 +806,8 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response
collection, ttl = await get_collectioninfo_for_id(collection_name)
if ttl is not None:
response.headers["cache-control"] = generate_max_age(ttl)
+ if collection is None:
+ raise EdrException(code=400, description="Unknown or unconfigured collection")
return collection
@@ -780,10 +860,10 @@ async def get_capabilities(collname):
for layername, layerinfo in wms.contents.items():
layers[layername] = {
"name": layername,
+ "title": layerinfo.title,
"dimensions": {**layerinfo.dimensions},
"boundingBoxWGS84": layerinfo.boundingBoxWGS84,
}
-
return layers, ttl
@@ -1053,7 +1133,7 @@ def makedims(dims, data):
return dimlist
-def covjson_from_resp(dats, vertical_name):
+def covjson_from_resp(dats, vertical_name, collection_name):
"""
Returns a coverage json from a Adaguc WMS GetFeatureInfo request
"""
@@ -1090,13 +1170,19 @@ def covjson_from_resp(dats, vertical_name):
parameters: dict[str, CovJsonParameter] = {}
ranges = {}
+ param_metadata = get_param_metadata(dat["name"], collection_name)
+ symbol = CovJsonSymbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL)
+ unit = CovJsonUnit(symbol=symbol)
+ observed_property = CovJsonObservedProperty(id=param_metadata["observed_property_id"], label={"en":param_metadata["observed_property_label"]})
- unit = CovJsonUnit(symbol=dat["units"])
param = CovJsonParameter(
id=dat["name"],
- observedProperty=CovJsonObservedProperty(label={"en": dat["name"]}),
+ observedProperty=observed_property,
+ # description={"en":param_metadata["wms_layer_title"]}, # TODO in follow up
unit=unit,
+ label={"en:":param_metadata["parameter_label"]}
)
+
parameters[dat["name"]] = param
axis_names = ["x", "y", "t"]
shape = [1, 1, len(time_steps)]
@@ -1164,7 +1250,7 @@ def covjson_from_resp(dats, vertical_name):
]
domain = Domain(domainType=domain_type, axes=axes, referencing=referencing)
covjson = Coverage(
- id="test",
+ id=f"coverage_{(len(covjson_list)+1)}",
domain=domain,
ranges=ranges,
parameters=parameters,
@@ -1178,32 +1264,6 @@ def covjson_from_resp(dats, vertical_name):
coverage_collection = CoverageCollection(coverages=covjson_list)
- # coverages = []
- # covjson_list.sort(key=lambda x: x.model_dump_json())
- # for _domain, group in itertools.groupby(covjson_list, lambda x: x.domain):
- # _ranges = {}
- # _parameters = {}
- # for cov in group:
- # param_id = next(iter(cov.parameters))
- # _parameters[param_id] = cov.parameters[param_id]
- # _ranges[param_id] = cov.ranges[param_id]
- # coverages.append(
- # Coverage(
- # domain=_domain,
- # ranges=_ranges,
- # parameters=_parameters,
- # )
- # )
- # if len(coverages) == 1:
- # return coverages[0]
-
- # parameter_union = functools.reduce(
- # operator.ior, (c.parameters for c in coverages), {}
- # )
- # coverage_collection = CoverageCollection(
- # coverages=coverages, parameters=parameter_union
- # )
-
return coverage_collection
diff --git a/python/python_fastapi_server/test_ogc_api_edr.py b/python/python_fastapi_server/test_ogc_api_edr.py
index fd5c72a8..e1f2f45d 100644
--- a/python/python_fastapi_server/test_ogc_api_edr.py
+++ b/python/python_fastapi_server/test_ogc_api_edr.py
@@ -71,6 +71,51 @@ def test_collections(client: TestClient):
assert "position" in coll_5d["data_queries"]
+ assert "parameter_names" in coll_5d
+
+ parameter_names = coll_5d["parameter_names"]
+
+ assert "data" in parameter_names
+
+ data = parameter_names["data"]
+
+ assert data == {
+ "type": "Parameter",
+ "id": "data",
+ "label": "data",
+ "unit": {
+ "symbol": {
+ "value": "unit",
+ "type": "http://www.opengis.net/def/uom/UCUM"
+ }
+ },
+ "observedProperty": {
+ "id": "data",
+ "label": "data"
+ }
+ }
+
+
+ assert "data_extra_metadata" in parameter_names
+
+ data_extra_metadata = parameter_names["data_extra_metadata"]
+
+ assert data_extra_metadata == {
+ "type": "Parameter",
+ "id": "data_extra_metadata",
+ "label": "Air temperature, 2 metre",
+ "unit": {
+ "symbol": {
+ "value": "\u00b0C",
+ "type": "http://www.opengis.net/def/uom/UCUM"
+ }
+ },
+ "observedProperty": {
+ "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature",
+ "label": "Air temperature"
+ }
+ }
+
def test_coll_5d_position(client: TestClient):
resp = client.get(