From 7a49aa0e7920a41d460847f61ebac59b600ce012 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Wed, 22 May 2024 12:45:18 +0200 Subject: [PATCH 1/9] EDR Metadata can now be harmonized --- Dockerfile | 2 +- NEWS.md | 4 ++ adagucserverEC/Definitions.h | 2 +- doc/tutorials/Configure_EDR_service.md | 7 ++- python/python_fastapi_server/routers/edr.py | 70 ++++++++++++++++++--- 5 files changed, 73 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5ef77c10..93cb5a2f 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.21.2" +LABEL version="2.21.3" # Try to update image packages RUN apt-get -q -y update \ diff --git a/NEWS.md b/NEWS.md index 7ebb35d3..155369e3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +**Version 2.21.3 2024-05-22** +- EDR Metadata can now be harmonized: https://github.com/KNMI/adaguc-server/issues/359 + + **Version 2.21.2 2024-04-26** - When using docker compose the Redis container now automatically starts when system restarts diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index e218206c..6b3188b4 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.21.2" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.21.3" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md index a396a24d..dcaf4399 100644 --- a/doc/tutorials/Configure_EDR_service.md +++ b/doc/tutorials/Configure_EDR_service.md @@ -23,14 +23,14 @@ This file is available in the adaguc-server repository with location `data/datas Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can also consider changing `` to `/data/adaguc-data/*.nc`. ```xml - + - + @@ -43,12 +43,13 @@ Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can + + air_temperature__at_2m /data/adaguc-data/HARM_N25_20171215090000_dimx16_dimy16_dimtime49_dimforecastreferencetime1_varairtemperatureat2m.nc air_temperature__at_2m temperature - diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 698aab11..26e9d9e7 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -15,6 +15,7 @@ import re from datetime import datetime, timezone from typing import Union +from xml.etree.ElementTree import Element from covjson_pydantic.coverage import Coverage from covjson_pydantic.domain import Domain, ValuesAxis @@ -42,6 +43,8 @@ from edr_pydantic.observed_property import ObservedProperty from edr_pydantic.parameter import Parameter from edr_pydantic.unit import Unit +from edr_pydantic.unit import Symbol + from edr_pydantic.variables import Variables @@ -64,6 +67,9 @@ OWSLIB_DUMMY_URL = "http://localhost:8000" +SYMBOL_TYPE_URL = "http://www.opengis.net/def/uom/UCUM" +VOCAB_ENDPOINT_URL = "https://vocab.nerc.ac.uk/standard_name/" + def get_base_url(req: Request = None) -> str: """Returns the base url of this service""" @@ -99,6 +105,33 @@ async def edr_exception_handler(_, exc: EdrException): ) +def find_layer_configuration(layers: Element, name: str): + """ + Checks if the layer is available as WMS layer in the adaguc dataset configuration by checking the layer name or variable + + Args: + layers (Element): Layer element from the adaguc dataset + name (str): The layer name or layer variable to find + + Returns: + _type_: Adaguc-server Layer element + """ + for layer in layers: + # First try to find the layer based on its name + layer_name_element = layer.find("Name") + if layer_name_element is not None: + layer_name = layer_name_element.text + if name == layer_name: + return layer + # Second try to find the layer by its variable + layer_variable_element = layer.find("Variable") + if layer_variable_element is not None: + layer_variable = layer_variable_element.text + if name == layer_variable: + return layer + return None + + def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DIR"]): """ Return all possible OGCAPI EDR datasets, based on the dataset directory @@ -115,6 +148,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI try: tree = parse(os.path.join(adaguc_dataset_dir, dataset_file)) root = tree.getroot() + adaguc_dataset_layers = root.iter("Layer") for ogcapi_edr in root.iter("OgcApiEdr"): for edr_collection in ogcapi_edr.iter("EdrCollection"): edr_params = [] @@ -123,12 +157,29 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI "name" in edr_parameter.attrib and "unit" in edr_parameter.attrib ): - edr_params.append( - { - "name": edr_parameter.attrib.get("name"), + name = edr_parameter.attrib.get("name") + layer = find_layer_configuration( + adaguc_dataset_layers, name + ) + # If the layer is present in the adaguc dataset configuration, continue. + if layer is not None: + edr_param = { + "name": name, "unit": edr_parameter.attrib.get("unit"), } - ) + # Try to take the standard name from the configuration + if "standard_name" in edr_parameter.attrib: + edr_param["standard_name"] = ( + edr_parameter.attrib.get("standard_name") + ) + # Try to take the observered propertylabel from the configuration + if "label" in edr_parameter.attrib: + edr_param["label"] = edr_parameter.attrib.get( + "label" + ) + + edr_params.append(edr_param) + else: logger.warning( "In dataset %s, skipping parameter %s: has no name or units configured", @@ -161,7 +212,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI # The edr_collections information is cached locally for a maximum of 2 minutes # It will be refreshed if older than 2 minutes -edr_cache = TTLCache(maxsize=100, ttl=120) +edr_cache = TTLCache(maxsize=100, ttl=120) # TODO: Redis? @cached(cache=edr_cache) @@ -420,11 +471,16 @@ def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]: else: label = param_el["name"] + symbol = Symbol(value=param_el["unit"], type=SYMBOL_TYPE_URL) + param_id = param_el["name"] + # If standard name was configured, use that instead with a vocabulary + if "standard_name" in param_el: + param_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] param = Parameter( id=param_el["name"], - observedProperty=ObservedProperty(id=param_el["name"], label=label), + observedProperty=ObservedProperty(id=param_id, label=label), type="Parameter", - unit=Unit(symbol=param_el["unit"]), + unit=Unit(symbol=symbol), label=label, ) parameter_names[param_el["name"]] = param From 7fc9c85fff7b7e3862af338b604df1cfd977cb6b Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 13:50:54 +0200 Subject: [PATCH 2/9] Resolved comments --- python/python_fastapi_server/routers/edr.py | 110 ++++++++++++-------- 1 file changed, 67 insertions(+), 43 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 26e9d9e7..41ac19ff 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -148,7 +148,6 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI try: tree = parse(os.path.join(adaguc_dataset_dir, dataset_file)) root = tree.getroot() - adaguc_dataset_layers = root.iter("Layer") for ogcapi_edr in root.iter("OgcApiEdr"): for edr_collection in ogcapi_edr.iter("EdrCollection"): edr_params = [] @@ -158,27 +157,40 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI and "unit" in edr_parameter.attrib ): name = edr_parameter.attrib.get("name") - layer = find_layer_configuration( - adaguc_dataset_layers, name - ) - # If the layer is present in the adaguc dataset configuration, continue. - if layer is not None: - edr_param = { - "name": name, - "unit": edr_parameter.attrib.get("unit"), - } - # Try to take the standard name from the configuration - if "standard_name" in edr_parameter.attrib: - edr_param["standard_name"] = ( - edr_parameter.attrib.get("standard_name") - ) - # Try to take the observered propertylabel from the configuration - if "label" in edr_parameter.attrib: - edr_param["label"] = edr_parameter.attrib.get( - "label" - ) - - edr_params.append(edr_param) + + edr_param = { + "name": name, + "parameter_label":name, + "observed_property_label":name, + "description":name, + "standard_name":None, + "unit": "-", + } + # Try to take the standard name from the configuration + if "standard_name" in edr_parameter.attrib: + edr_param["standard_name"] = ( + edr_parameter.attrib.get("standard_name") + ) + # If there is a standard name, set the label to the same as fallback + edr_param["observed_property_label"] = ( + edr_parameter.attrib.get("standard_name") + ) + + # Try to take the oobserved_property_label from the configuration + if "observed_property_label" in edr_parameter.attrib: + edr_param["observed_property_label"] = edr_parameter.attrib.get( + "observed_property_label" + ) + # Try to take the parameter_label from the Layer configuration Title + if "parameter_label" in edr_parameter.attrib: + edr_param["parameter_label"] = edr_parameter.attrib.get( + "parameter_label") + # Try to take the parameter_label from the Layer configuration Title + if "unit" in edr_parameter.attrib: + edr_param["unit"] = edr_parameter.attrib.get( + "unit") + + edr_params.append(edr_param) else: logger.warning( @@ -212,7 +224,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI # The edr_collections information is cached locally for a maximum of 2 minutes # It will be refreshed if older than 2 minutes -edr_cache = TTLCache(maxsize=100, ttl=120) # TODO: Redis? +edr_cache = TTLCache(maxsize=100, ttl=0) # TODO: Redis? @cached(cache=edr_cache) @@ -330,7 +342,10 @@ async def get_collectioninfo_for_id( Is used to obtain metadata from the dataset configuration and WMS GetCapabilities document. """ logger.info("get_collectioninfo_for_id(%s, %s)", edr_collection, instance) - edr_collectioninfo = get_edr_collections()[edr_collection] + edr_collectioninfo = get_edr_collections().get(edr_collection) + if edr_collectioninfo is None: + raise EdrException(code=400, description="Unknown or unconfigured collection") + dataset = edr_collectioninfo["dataset"] logger.info("%s=>%s", edr_collection, dataset) @@ -429,7 +444,7 @@ async def get_collectioninfo_for_id( else: data_queries = DataQueries(position=EDRQuery(link=position_link)) - parameter_names = get_params_for_collection(edr_collection=edr_collection) + parameter_names = get_params_for_collection(edr_collection=edr_collection, wmslayers=wmslayers) crs = ["EPSG:4326"] @@ -458,32 +473,38 @@ async def get_collectioninfo_for_id( return collection, ttl -def get_params_for_collection(edr_collection: str) -> dict[str, Parameter]: +def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, Parameter]: """ Returns a dictionary with parameters for given EDR collection """ parameter_names = {} edr_collections = get_edr_collections() for param_el in edr_collections[edr_collection]["parameters"]: - # Use name as default for label - if "label" in param_el: - label = param_el["label"] - else: - label = param_el["name"] symbol = Symbol(value=param_el["unit"], type=SYMBOL_TYPE_URL) param_id = param_el["name"] - # If standard name was configured, use that instead with a vocabulary - if "standard_name" in param_el: - param_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] - param = Parameter( - id=param_el["name"], - observedProperty=ObservedProperty(id=param_id, label=label), - type="Parameter", - unit=Unit(symbol=symbol), - label=label, - ) - parameter_names[param_el["name"]] = param + + + if not param_id in wmslayers: + logger.warning("EDR Parameter with name [%s] is not found in any of the adaguc Layer configurations. Available layers are %s", param_id, str(list(wmslayers.keys()))) + else: + wmslayer = wmslayers[param_id] + wms_layer_name = param_el["name"] + wms_layer_title = wmslayer["title"] + parameter_label = param_el["parameter_label"] + observed_property_label = param_el["observed_property_label"] + # If standard name was configured, use that instead with a vocabulary + if "standard_name" in param_el and param_el["standard_name"] is not None: + param_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] + param = Parameter( + id=wms_layer_name, + observedProperty=ObservedProperty(id=param_id, label=observed_property_label), + description=wms_layer_title, + type="Parameter", + unit=Unit(symbol=symbol), + label=parameter_label + ) + parameter_names[param_el["name"]] = param return parameter_names @@ -690,6 +711,8 @@ async def rest_get_edr_collection_by_id(collection_name: str, response: Response collection, ttl = await get_collectioninfo_for_id(collection_name) if ttl is not None: response.headers["cache-control"] = generate_max_age(ttl) + if collection is None: + raise EdrException(code=400, description="Unknown or unconfigured collection") return collection @@ -746,10 +769,11 @@ async def get_capabilities(collname): for layername, layerinfo in wms.contents.items(): layers[layername] = { "name": layername, + "title": layerinfo.title, "dimensions": {**layerinfo.dimensions}, "boundingBoxWGS84": layerinfo.boundingBoxWGS84, } - + return layers, ttl From 8a60610653966921efb22e8b859596ebb1227935 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 14:00:04 +0200 Subject: [PATCH 3/9] Updated documentation --- doc/tutorials/Configure_EDR_service.md | 57 ++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 4 deletions(-) diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md index dcaf4399..87e4aaa3 100644 --- a/doc/tutorials/Configure_EDR_service.md +++ b/doc/tutorials/Configure_EDR_service.md @@ -26,11 +26,16 @@ Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can - + - + @@ -45,17 +50,61 @@ Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can - air_temperature__at_2m - /data/adaguc-data/HARM_N25_20171215090000_dimx16_dimy16_dimtime49_dimforecastreferencetime1_varairtemperatureat2m.nc + + + /data/adaguc-data/HARM_N25_20171215090000_dimx16_dimy16_dimtime49_dimforecastreferencetime1_varairtemperatureat2m.nc air_temperature__at_2m temperature +``` + + +### EdrParameter settings +```xml + +``` + +- name: Mandatory, Should be one of the WMS Layer names as advertised in the WMS GetCapabilities +- unit: Mandatory, Sets the unit for the parameter in the parameter_names section of the collection document +- standard_name: Sets the observedProperty id +- observed_property_label: Sets the observedProperty label +- parameter_label: Sets the label for the parameter in the parameter_names section + +For the given example this will result in the following parameter name definition: + +```json +parameter_names": { + "air_temperature__at_2m": { + "type": "Parameter", + "id": "air_temperature__at_2m", + "label": "Air temperature, 2 metre", + "description": "air_temperature__at_2m (air_temperature__at_2m)", + "unit": { + "symbol": { + "value": "°C", + "type": "http://www.opengis.net/def/uom/UCUM" + } + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": "Air temperature" + } + } + } ``` +*The description is currently read from the GetCapabilities document, using the WMS Layer Title section . + + ## Step 3: Scan the new data ``` From a116463b564fef34dac66dc105abcdaff3dcd095 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 14:10:29 +0200 Subject: [PATCH 4/9] Updated documentation --- doc/tutorials/Configure_EDR_service.md | 4 ++-- python/python_fastapi_server/routers/edr.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md index 87e4aaa3..0b0a91ef 100644 --- a/doc/tutorials/Configure_EDR_service.md +++ b/doc/tutorials/Configure_EDR_service.md @@ -87,7 +87,7 @@ parameter_names": { "type": "Parameter", "id": "air_temperature__at_2m", "label": "Air temperature, 2 metre", - "description": "air_temperature__at_2m (air_temperature__at_2m)", + "description": "harmonie - air_temperature__at_2m (air_temperature__at_2m)", "unit": { "symbol": { "value": "°C", @@ -102,7 +102,7 @@ parameter_names": { } ``` -*The description is currently read from the GetCapabilities document, using the WMS Layer Title section . +*The description is currently read from the GetCapabilities document, using the WMS Layer Title section prefixed with the dataset name. ## Step 3: Scan the new data diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 41ac19ff..25a4160f 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -490,7 +490,7 @@ def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, else: wmslayer = wmslayers[param_id] wms_layer_name = param_el["name"] - wms_layer_title = wmslayer["title"] + wms_layer_title = edr_collection+" - "+ wmslayer["title"] parameter_label = param_el["parameter_label"] observed_property_label = param_el["observed_property_label"] # If standard name was configured, use that instead with a vocabulary From c5e99fd8376d9fd0f4197e46e0789253bf3080fc Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 14:14:39 +0200 Subject: [PATCH 5/9] Updated documentation --- Dockerfile | 2 +- NEWS.md | 5 +++-- adagucserverEC/Definitions.h | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 93cb5a2f..d28621fd 100755 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ USER root LABEL maintainer="adaguc@knmi.nl" # Version should be same as in Definitions.h -LABEL version="2.21.3" +LABEL version="2.22.0" # Try to update image packages RUN apt-get -q -y update \ diff --git a/NEWS.md b/NEWS.md index 155369e3..acbc8592 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,6 @@ -**Version 2.21.3 2024-05-22** -- EDR Metadata can now be harmonized: https://github.com/KNMI/adaguc-server/issues/359 +**Version 2.22.0 2024-05-22** +- EDR Metadata can now be harmonized: https://github.com/KNMI/adaguc-server/issues/359. +- See [Configure_EDR_service](doc/tutorials/Configure_EDR_service.md) for details. **Version 2.21.2 2024-04-26** diff --git a/adagucserverEC/Definitions.h b/adagucserverEC/Definitions.h index 6b3188b4..14d8ca80 100755 --- a/adagucserverEC/Definitions.h +++ b/adagucserverEC/Definitions.h @@ -28,7 +28,7 @@ #ifndef Definitions_H #define Definitions_H -#define ADAGUCSERVER_VERSION "2.21.3" // Please also update in the Dockerfile to the same version +#define ADAGUCSERVER_VERSION "2.22.0" // Please also update in the Dockerfile to the same version // CConfigReaderLayerType #define CConfigReaderLayerTypeUnknown 0 From 5d5ab0a5c3e598cd39695aad76811983b1ade4b2 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 14:16:19 +0200 Subject: [PATCH 6/9] Updated documentation --- doc/tutorials/Configure_EDR_service.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md index 0b0a91ef..f759acd6 100644 --- a/doc/tutorials/Configure_EDR_service.md +++ b/doc/tutorials/Configure_EDR_service.md @@ -102,7 +102,7 @@ parameter_names": { } ``` -*The description is currently read from the GetCapabilities document, using the WMS Layer Title section prefixed with the dataset name. +*The description is currently read from the GetCapabilities document, using the WMS Layer Title section prefixed with the edr collection name. ## Step 3: Scan the new data From 270474530c2afad0f743140a1d781573b6af8bc3 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 23 May 2024 15:39:38 +0200 Subject: [PATCH 7/9] EDR metadata is now also updated for the position query --- python/python_fastapi_server/routers/edr.py | 95 ++++++++++++++++----- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 25a4160f..78798156 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -28,6 +28,7 @@ ReferenceSystemConnectionObject, ) from covjson_pydantic.unit import Unit as CovJsonUnit +from covjson_pydantic.unit import Symbol as CovJsonSymbol from defusedxml.ElementTree import ParseError, parse from edr_pydantic.capabilities import ( @@ -224,7 +225,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI # The edr_collections information is cached locally for a maximum of 2 minutes # It will be refreshed if older than 2 minutes -edr_cache = TTLCache(maxsize=100, ttl=0) # TODO: Redis? +edr_cache = TTLCache(maxsize=100, ttl=120) # TODO: Redis? @cached(cache=edr_cache) @@ -322,7 +323,7 @@ async def get_collection_position( ttl = get_ttl_from_adaguc_headers(headers) if ttl is not None: response.headers["cache-control"] = generate_max_age(ttl) - return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"]) + return covjson_from_resp(dat, edr_collections[collection_name]["vertical_name"],collection_name) raise EdrException(code=400, description="No data") @@ -472,6 +473,64 @@ async def get_collectioninfo_for_id( return collection, ttl +def get_param_el(edr_collection_name:str, param_id:str): + """Gets the EDRParameter configuration based on the edr collection name and parameter id + + Args: + edr_collection_name (str): edr collection name + param_id (str): parameter id + + Returns: + _type_: parameter element_ + """ + edr_collections = get_edr_collections() + edr_collection_parameters= edr_collections[edr_collection_name]["parameters"] + for param_el in edr_collection_parameters: + param_id = param_el["name"] + if param_id == param_el["name"]: + return param_el + return None + +def get_param_metadata( param_id:str, edr_collection_name:str,wmslayers:dict=None)->dict: + """Composes parameter metadata based on the param_el and the wmslayer dictionaries + + Args: + wmslayers (dict): wmslayers list obtained from the WMS GetCapabilities document + param_id (str): The parameter / wms layer name to find + edr_collection_name (str): The collection name + + Returns: + dict: dictionary with all metadata required to construct a Edr Parameter object. + """ + + + param_el = get_param_el(edr_collection_name, param_id) + + + + + wmslayer_title = param_id + if wmslayers is not None and param_id in wmslayers: + wmslayer = wmslayers[param_id] + wmslayer_title = wmslayer["title"] + wms_layer_name = param_el["name"] + wms_layer_title = edr_collection_name+" - "+ wmslayer_title + observed_property_id = wms_layer_name + parameter_label = param_el["parameter_label"] + parameter_unit = param_el["unit"] + observed_property_label = param_el["observed_property_label"] + if "standard_name" in param_el and param_el["standard_name"] is not None: + observed_property_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] + + return { "wms_layer_name":wms_layer_name, + "wms_layer_title":wms_layer_title, + "observed_property_id":observed_property_id, + "observed_property_label":observed_property_label, + "parameter_label":parameter_label, + "parameter_unit":parameter_unit} + + + def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, Parameter]: """ @@ -481,28 +540,20 @@ def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, edr_collections = get_edr_collections() for param_el in edr_collections[edr_collection]["parameters"]: - symbol = Symbol(value=param_el["unit"], type=SYMBOL_TYPE_URL) param_id = param_el["name"] if not param_id in wmslayers: logger.warning("EDR Parameter with name [%s] is not found in any of the adaguc Layer configurations. Available layers are %s", param_id, str(list(wmslayers.keys()))) else: - wmslayer = wmslayers[param_id] - wms_layer_name = param_el["name"] - wms_layer_title = edr_collection+" - "+ wmslayer["title"] - parameter_label = param_el["parameter_label"] - observed_property_label = param_el["observed_property_label"] - # If standard name was configured, use that instead with a vocabulary - if "standard_name" in param_el and param_el["standard_name"] is not None: - param_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] + param_metadata = get_param_metadata(param_id, edr_collection, wmslayers) param = Parameter( - id=wms_layer_name, - observedProperty=ObservedProperty(id=param_id, label=observed_property_label), - description=wms_layer_title, + id=param_metadata["wms_layer_name"], + observedProperty=ObservedProperty(id=param_metadata["observed_property_id"], label=param_metadata["observed_property_label"]), + description=param_metadata["wms_layer_title"], type="Parameter", - unit=Unit(symbol=symbol), - label=parameter_label + unit=Unit(symbol=Symbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL)), + label=param_metadata["parameter_label"] ) parameter_names[param_el["name"]] = param return parameter_names @@ -1040,7 +1091,7 @@ def makedims(dims, data): return dimlist -def covjson_from_resp(dats, vertical_name): +def covjson_from_resp(dats, vertical_name, collection_name): """ Returns a coverage json from a Adaguc WMS GetFeatureInfo request """ @@ -1076,12 +1127,18 @@ def covjson_from_resp(dats, vertical_name): parameters: dict[str, CovJsonParameter] = {} ranges = {} + param_metadata = get_param_metadata(dat["name"], collection_name) - unit = CovJsonUnit(symbol=dat["units"]) + symbol = CovJsonSymbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL) + unit = CovJsonUnit(symbol=symbol) + observed_property = CovJsonObservedProperty(id=param_metadata["observed_property_id"], label={"en":param_metadata["observed_property_label"]}) + param = CovJsonParameter( id=dat["name"], - observedProperty=CovJsonObservedProperty(label={"en": dat["name"]}), + observedProperty=observed_property, + description={"en":param_metadata["wms_layer_title"]}, unit=unit, + label={"en:":param_metadata["parameter_label"]} ) # TODO: add units to CovJsonParameter parameters[dat["name"]] = param From 7dd4d1f5324021003b9cdafbeb021e032314eae3 Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Wed, 29 May 2024 18:04:44 +0200 Subject: [PATCH 8/9] Resolved comments --- NEWS.md | 4 +-- doc/tutorials/Configure_EDR_service.md | 6 ++--- python/python_fastapi_server/routers/edr.py | 30 ++++++--------------- 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/NEWS.md b/NEWS.md index acbc8592..babb84e5 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,6 +1,6 @@ **Version 2.22.0 2024-05-22** -- EDR Metadata can now be harmonized: https://github.com/KNMI/adaguc-server/issues/359. -- See [Configure_EDR_service](doc/tutorials/Configure_EDR_service.md) for details. +- EDR: Parameters can now be detailed with metadata like standard_names and units: https://github.com/KNMI/adaguc-server/issues/359. +- See [Configure_EDR_service](doc/tutorials/Configure_EDR_service.md) for details. **Version 2.21.2 2024-04-26** diff --git a/doc/tutorials/Configure_EDR_service.md b/doc/tutorials/Configure_EDR_service.md index f759acd6..5326bf76 100644 --- a/doc/tutorials/Configure_EDR_service.md +++ b/doc/tutorials/Configure_EDR_service.md @@ -75,9 +75,9 @@ Create the following file at the filepath `$ADAGUC_DATASET_DIR/edr.xml`. You can - name: Mandatory, Should be one of the WMS Layer names as advertised in the WMS GetCapabilities - unit: Mandatory, Sets the unit for the parameter in the parameter_names section of the collection document -- standard_name: Sets the observedProperty id -- observed_property_label: Sets the observedProperty label -- parameter_label: Sets the label for the parameter in the parameter_names section +- standard_name: Recommended, sets the observedProperty id. If set the id will contain a link to the vocabulary service. If not set, it will fallback to `name` and `observedProperty.id` will not contain a link. +- observed_property_label: Recommended, sets the observedProperty label, it will fallback to it will fallback first to `standard_name` first, and second to `name` +- parameter_label: Recommended, sets the label for the parameter in the parameter_names section, it will fallback to the `name` For the given example this will result in the following parameter name definition: diff --git a/python/python_fastapi_server/routers/edr.py b/python/python_fastapi_server/routers/edr.py index 47837748..f2bc5d47 100644 --- a/python/python_fastapi_server/routers/edr.py +++ b/python/python_fastapi_server/routers/edr.py @@ -151,7 +151,7 @@ def init_edr_collections(adaguc_dataset_dir: str = os.environ["ADAGUC_DATASET_DI edr_parameter.attrib.get("standard_name") ) - # Try to take the oobserved_property_label from the configuration + # Try to take the observed_property_label from the configuration if "observed_property_label" in edr_parameter.attrib: edr_param["observed_property_label"] = edr_parameter.attrib.get( "observed_property_label" @@ -475,7 +475,7 @@ async def get_collectioninfo_for_id( return collection, ttl -def get_param_el(edr_collection_name:str, param_id:str): +def get_parameter_config(edr_collection_name:str, param_id:str): """Gets the EDRParameter configuration based on the edr collection name and parameter id Args: @@ -486,30 +486,24 @@ def get_param_el(edr_collection_name:str, param_id:str): _type_: parameter element_ """ edr_collections = get_edr_collections() - edr_collection_parameters= edr_collections[edr_collection_name]["parameters"] + edr_collection_parameters = edr_collections[edr_collection_name]["parameters"] for param_el in edr_collection_parameters: if param_id == param_el["name"]: return param_el return None -def get_param_metadata( param_id:str, edr_collection_name:str,wmslayers:dict=None)->dict: +def get_param_metadata(param_id:str, edr_collection_name)->dict: """Composes parameter metadata based on the param_el and the wmslayer dictionaries Args: - wmslayers (dict): wmslayers list obtained from the WMS GetCapabilities document param_id (str): The parameter / wms layer name to find edr_collection_name (str): The collection name Returns: dict: dictionary with all metadata required to construct a Edr Parameter object. """ - param_el = get_param_el(edr_collection_name, param_id) - wmslayer_title = param_id - if wmslayers is not None and param_id in wmslayers: - wmslayer = wmslayers[param_id] - wmslayer_title = wmslayer["title"] + param_el = get_parameter_config(edr_collection_name, param_id) wms_layer_name = param_el["name"] - wms_layer_title = edr_collection_name+" - "+ wmslayer_title observed_property_id = wms_layer_name parameter_label = param_el["parameter_label"] parameter_unit = param_el["unit"] @@ -518,7 +512,6 @@ def get_param_metadata( param_id:str, edr_collection_name:str,wmslayers:dict=Non observed_property_id = VOCAB_ENDPOINT_URL + param_el["standard_name"] return { "wms_layer_name":wms_layer_name, - "wms_layer_title":wms_layer_title, "observed_property_id":observed_property_id, "observed_property_label":observed_property_label, "parameter_label":parameter_label, @@ -531,21 +524,17 @@ def get_params_for_collection(edr_collection: str, wmslayers: dict) -> dict[str, parameter_names = {} edr_collections = get_edr_collections() for param_el in edr_collections[edr_collection]["parameters"]: - param_id = param_el["name"] - - if not param_id in wmslayers: logger.warning("EDR Parameter with name [%s] is not found in any of the adaguc Layer configurations. Available layers are %s", param_id, str(list(wmslayers.keys()))) else: - - param_metadata = get_param_metadata(param_id, edr_collection, wmslayers) + param_metadata = get_param_metadata(param_id, edr_collection) logger.info(param_id) logger.info(param_metadata["wms_layer_name"]) param = Parameter( id=param_metadata["wms_layer_name"], observedProperty=ObservedProperty(id=param_metadata["observed_property_id"], label=param_metadata["observed_property_label"]), - description=param_metadata["wms_layer_title"], + # description=param_metadata["wms_layer_title"], # TODO in follow up type="Parameter", unit=Unit(symbol=Symbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL)), label=param_metadata["parameter_label"] @@ -1181,10 +1170,7 @@ def covjson_from_resp(dats, vertical_name, collection_name): parameters: dict[str, CovJsonParameter] = {} ranges = {} - - param_metadata = get_param_metadata(dat["name"], collection_name) - symbol = CovJsonSymbol(value=param_metadata["parameter_unit"], type=SYMBOL_TYPE_URL) unit = CovJsonUnit(symbol=symbol) observed_property = CovJsonObservedProperty(id=param_metadata["observed_property_id"], label={"en":param_metadata["observed_property_label"]}) @@ -1192,7 +1178,7 @@ def covjson_from_resp(dats, vertical_name, collection_name): param = CovJsonParameter( id=dat["name"], observedProperty=observed_property, - description={"en":param_metadata["wms_layer_title"]}, + # description={"en":param_metadata["wms_layer_title"]}, # TODO in follow up unit=unit, label={"en:":param_metadata["parameter_label"]} ) From 9f4147cf88c236ce8e1cc02aae6a0f49f46d2faf Mon Sep 17 00:00:00 2001 From: Maarten Plieger Date: Thu, 6 Jun 2024 16:17:23 +0200 Subject: [PATCH 9/9] Added tests for extra edr metadata --- data/config/datasets/netcdf_5d.xml | 15 +++++++ .../python_fastapi_server/test_ogc_api_edr.py | 45 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/data/config/datasets/netcdf_5d.xml b/data/config/datasets/netcdf_5d.xml index fec34846..062fcb27 100644 --- a/data/config/datasets/netcdf_5d.xml +++ b/data/config/datasets/netcdf_5d.xml @@ -5,6 +5,13 @@ + + @@ -38,6 +45,14 @@ + + data_extra_metadata + data_extra_metadata + {ADAGUC_PATH}/data/datasets/netcdf_5dims + data + testdata + + diff --git a/python/python_fastapi_server/test_ogc_api_edr.py b/python/python_fastapi_server/test_ogc_api_edr.py index fd5c72a8..e1f2f45d 100644 --- a/python/python_fastapi_server/test_ogc_api_edr.py +++ b/python/python_fastapi_server/test_ogc_api_edr.py @@ -71,6 +71,51 @@ def test_collections(client: TestClient): assert "position" in coll_5d["data_queries"] + assert "parameter_names" in coll_5d + + parameter_names = coll_5d["parameter_names"] + + assert "data" in parameter_names + + data = parameter_names["data"] + + assert data == { + "type": "Parameter", + "id": "data", + "label": "data", + "unit": { + "symbol": { + "value": "unit", + "type": "http://www.opengis.net/def/uom/UCUM" + } + }, + "observedProperty": { + "id": "data", + "label": "data" + } + } + + + assert "data_extra_metadata" in parameter_names + + data_extra_metadata = parameter_names["data_extra_metadata"] + + assert data_extra_metadata == { + "type": "Parameter", + "id": "data_extra_metadata", + "label": "Air temperature, 2 metre", + "unit": { + "symbol": { + "value": "\u00b0C", + "type": "http://www.opengis.net/def/uom/UCUM" + } + }, + "observedProperty": { + "id": "https://vocab.nerc.ac.uk/standard_name/air_temperature", + "label": "Air temperature" + } + } + def test_coll_5d_position(client: TestClient): resp = client.get(