From bc37fc42d1d26647a4d65e1fccff254c160c9a41 Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 25 Mar 2024 10:02:51 -0500 Subject: [PATCH 01/10] adding radar parameters to xradar iris datatree --- xradar/io/backends/iris.py | 90 +++++++++++++++++++++++++++++++++----- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 07a5fdbe..18437fc7 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -61,6 +61,11 @@ get_range_attrs, moment_attrs, sweep_vars_mapping, + required_root_vars, + optional_root_vars, + required_global_attrs, + optional_root_attrs, + radar_parameters_subgroup, ) from .common import _assign_root, _attach_sweep_groups @@ -981,7 +986,6 @@ def array_dict(size, dtype): ] ) - # status_antenna_info Structure # 4.3.40, page 51 STATUS_ANTENNA_INFO = OrderedDict( @@ -1697,7 +1701,6 @@ def array_dict(size, dtype): ] ) - # some length's of data structures LEN_STRUCTURE_HEADER = struct.calcsize(_get_fmt_string(STRUCTURE_HEADER)) LEN_PRODUCT_HDR = struct.calcsize(_get_fmt_string(PRODUCT_HDR)) @@ -2466,7 +2469,6 @@ def array_dict(size, dtype): ] ) - PRODUCT_DATA_TYPE_CODES = OrderedDict( [ (0, {"name": "NULL", "struct": SPARE_PSI_STRUCT}), @@ -2529,7 +2531,6 @@ def array_dict(size, dtype): ] ) - RECORD_BYTES = 6144 @@ -3951,10 +3952,72 @@ def _get_iris_group_names(filename): sid, opener = _check_iris_file(filename) with opener(filename, loaddata=False) as ds: # make this CfRadial2 conform - keys = [f"sweep_{i-1}" for i in list(ds.data.keys())] + keys = [f"sweep_{i - 1}" for i in list(ds.data.keys())] return keys +def _get_required_root_dataset(ls_ds, optional=True): + """Extract Root Dataset.""" + # keep only defined mandatory and defined optional variables per default + data_var = set( + [x for xs in [sweep.variables.keys() for sweep in ls_ds] for x in xs] + ) + # new_req_root_vars = [var for var in required_root_vars if var not in ['sweep_group_name', 'sweep_fixed_angle']] + remove_root = set(data_var) ^ set(required_root_vars) + if optional: + remove_root ^= set(optional_root_vars) + remove_root ^= { + "sweep_number", + "fixed_angle", + "sweep_group_name", + "sweep_fixed_angle", + } + # remove_root ^= {"sweep_number", "fixed_angle", "sweep_group_name", "sweep_fixed_angle"} + remove_root &= data_var + root = [sweep.drop_vars(remove_root) for sweep in ls_ds] + root_vars = set( + [x for xs in [sweep.variables.keys() for sweep in root] for x in xs] + ) + # rename variables + # todo: find a more easy method not iterating over all variables + for k in root_vars: + rename = optional_root_vars.get(k, None) + if rename: + root = [sweep.rename_vars({k: rename}) for sweep in root] + + root_vars = set( + [x for xs in [sweep.variables.keys() for sweep in root] for x in xs] + ) + ds_vars = [sweep[root_vars] for sweep in ls_ds] + + root = xr.concat(ds_vars, dim="sweep") + # keep only defined mandatory and defined optional attributes per default + attrs = root.attrs.keys() + remove_attrs = set(attrs) ^ set(required_global_attrs) + if optional: + remove_attrs ^= set(optional_root_attrs) + for k in remove_attrs: + root.attrs.pop(k, None) + return root + + +def _get_subgroup(ls_ds: list[xr.Dataset], subdict): + """Get iris-sigmet root metadata group. + Variables are fetched from the provided Dataset according to the subdict dictionary. + """ + print(1) + meta_vars = subdict + data_vars = set([x for xs in [ds.variables.keys() for ds in ls_ds] for x in xs]) + extract_vars = set(data_vars) & set(meta_vars) + subgroup = xr.concat([ds[extract_vars] for ds in ls_ds], "sweep") + for k in subgroup.data_vars: + rename = meta_vars[k] + if rename: + subgroup = subgroup.rename_vars({k: rename}) + subgroup.attrs = {} + return subgroup + + class IrisBackendEntrypoint(BackendEntrypoint): """Xarray BackendEntrypoint for IRIS/Sigmet data.""" @@ -4092,14 +4155,17 @@ def open_iris_datatree(filename_or_obj, **kwargs): else: sweeps = _get_iris_group_names(filename_or_obj) - ds = [ + ls_ds: list[xr.Dataset] = [ xr.open_dataset(filename_or_obj, group=swp, engine="iris", **kwargs) for swp in sweeps ] - - ds.insert(0, xr.Dataset()) - + # get the datatree root + root = _get_required_root_dataset(ls_ds) # create datatree root node with required data - dtree = DataTree(data=_assign_root(ds), name="root") - # return datatree with attached sweep child nodes - return _attach_sweep_groups(dtree, ds[1:]) + dtree = DataTree(data=root, name="root") + # get radar_parameters group + subgroup = _get_subgroup(ls_ds, radar_parameters_subgroup) + # attach radar_parameter group + DataTree(subgroup, name="radar_parameters", parent=dtree) + # return Datatree attaching the sweep child nodes + return _attach_sweep_groups(dtree, ls_ds) From f323783db417df976a03e56ab718c673488407af Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 25 Mar 2024 13:08:21 -0500 Subject: [PATCH 02/10] running ruff and fixing imports --- xradar/io/backends/iris.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 18437fc7..6e6473f9 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -60,14 +60,14 @@ get_longitude_attrs, get_range_attrs, moment_attrs, - sweep_vars_mapping, - required_root_vars, - optional_root_vars, - required_global_attrs, optional_root_attrs, + optional_root_vars, radar_parameters_subgroup, + required_global_attrs, + required_root_vars, + sweep_vars_mapping, ) -from .common import _assign_root, _attach_sweep_groups +from .common import _attach_sweep_groups #: mapping from IRIS names to CfRadial2/ODIM iris_mapping = { From 703287fa594ed00100cf8a2737054e7eed33f63f Mon Sep 17 00:00:00 2001 From: aladinor Date: Mon, 25 Mar 2024 15:22:50 -0500 Subject: [PATCH 03/10] get ride of print statement --- xradar/io/backends/iris.py | 1 - 1 file changed, 1 deletion(-) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 6e6473f9..8346d1f6 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -4005,7 +4005,6 @@ def _get_subgroup(ls_ds: list[xr.Dataset], subdict): """Get iris-sigmet root metadata group. Variables are fetched from the provided Dataset according to the subdict dictionary. """ - print(1) meta_vars = subdict data_vars = set([x for xs in [ds.variables.keys() for ds in ls_ds] for x in xs]) extract_vars = set(data_vars) & set(meta_vars) From f7d98cafad554f00d74e22d15b6bfd6067e7016d Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 09:56:39 -0500 Subject: [PATCH 04/10] adding global attributes and varibles to root dataset following tables Table 301-1 and 301-4a in the FM 301-2022 WMO-CF RADIALREGULATIONS document --- xradar/io/backends/iris.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 8346d1f6..81b83035 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -67,7 +67,7 @@ required_root_vars, sweep_vars_mapping, ) -from .common import _attach_sweep_groups +from .common import _attach_sweep_groups, _assign_root #: mapping from IRIS names to CfRadial2/ODIM iris_mapping = { @@ -3962,17 +3962,14 @@ def _get_required_root_dataset(ls_ds, optional=True): data_var = set( [x for xs in [sweep.variables.keys() for sweep in ls_ds] for x in xs] ) - # new_req_root_vars = [var for var in required_root_vars if var not in ['sweep_group_name', 'sweep_fixed_angle']] remove_root = set(data_var) ^ set(required_root_vars) if optional: remove_root ^= set(optional_root_vars) remove_root ^= { - "sweep_number", "fixed_angle", "sweep_group_name", "sweep_fixed_angle", } - # remove_root ^= {"sweep_number", "fixed_angle", "sweep_group_name", "sweep_fixed_angle"} remove_root &= data_var root = [sweep.drop_vars(remove_root) for sweep in ls_ds] root_vars = set( @@ -3990,7 +3987,7 @@ def _get_required_root_dataset(ls_ds, optional=True): ) ds_vars = [sweep[root_vars] for sweep in ls_ds] - root = xr.concat(ds_vars, dim="sweep") + root = xr.concat(ds_vars, dim="sweep").reset_coords() # keep only defined mandatory and defined optional attributes per default attrs = root.attrs.keys() remove_attrs = set(attrs) ^ set(required_global_attrs) @@ -3998,6 +3995,13 @@ def _get_required_root_dataset(ls_ds, optional=True): remove_attrs ^= set(optional_root_attrs) for k in remove_attrs: root.attrs.pop(k, None) + # creating a copy of the dataset list for using the _assing_root function. + # and get the variabes/attributes for the root dataset + ls = ls_ds.copy() + ls.insert(0, xr.Dataset()) + dtree = DataTree(data=_assign_root(ls), name="root") + root = root.assign(dtree.variables) + root.attrs = dtree.attrs return root From 6731fae676a125a5780d4e18018e94dad5936f16 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 09:59:06 -0500 Subject: [PATCH 05/10] As as the root datatree now includes radar_parameters and other datasets, test_open_iris_datatree function was refactored to check the only the sweeps dataset within the iris datatree --- tests/io/test_io.py | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 1d41b547..66697cc7 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -616,26 +616,29 @@ def test_open_iris_datatree(iris0_file): ] azimuths = [360] * 10 ranges = [664] * 10 - for i, grp in enumerate(dtree.groups[1:]): - ds = dtree[grp].ds - assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]} - assert set(ds.data_vars) & ( - sweep_dataset_vars | non_standard_sweep_dataset_vars - ) == set(moments) - assert set(ds.data_vars) & (required_sweep_metadata_vars) == set( - required_sweep_metadata_vars ^ {"azimuth", "elevation"} - ) - assert set(ds.coords) == { - "azimuth", - "elevation", - "time", - "latitude", - "longitude", - "altitude", - "range", - } - assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i] - assert ds.sweep_number == i + i = 0 + for grp in dtree.groups: + if grp.startswith('/sweep_'): + ds = dtree[grp].ds + assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]} + assert set(ds.data_vars) & ( + sweep_dataset_vars | non_standard_sweep_dataset_vars + ) == set(moments) + assert set(ds.data_vars) & (required_sweep_metadata_vars) == set( + required_sweep_metadata_vars ^ {"azimuth", "elevation"} + ) + assert set(ds.coords) == { + "azimuth", + "elevation", + "time", + "latitude", + "longitude", + "altitude", + "range", + } + assert np.round(ds.elevation.mean().values.item(), 1) == elevations[i] + assert ds.sweep_number == i + i += 1 def test_open_iris0_dataset(iris0_file): From 98d5e1fa6e4e5b95f0e9f2b993779d1ce907c451 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 10:01:35 -0500 Subject: [PATCH 06/10] refactoring given the new iris datatree structure --- .../multiple-sweeps-into-volume-scan.ipynb | 22 +++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/examples/notebooks/multiple-sweeps-into-volume-scan.ipynb b/examples/notebooks/multiple-sweeps-into-volume-scan.ipynb index bd90c714..48c32967 100644 --- a/examples/notebooks/multiple-sweeps-into-volume-scan.ipynb +++ b/examples/notebooks/multiple-sweeps-into-volume-scan.ipynb @@ -144,9 +144,10 @@ " sweeps = list(i.children.keys())\n", " print(f\"task sweeps: {sweeps}\")\n", " for j in sweeps:\n", - " print(\n", - " f\"{j}: {i[j].sweep_fixed_angle.values: .1f} [deg], {i[j].range.values[-1] / 1e3:.1f} [km]\"\n", - " )\n", + " if j.startswith(\"sweep\"):\n", + " print(\n", + " f\"{j}: {i[j].sweep_fixed_angle.values: .1f} [deg], {i[j].range.values[-1] / 1e3:.1f} [km]\"\n", + " )\n", " print(\"----------------------------------------------------------------\")" ] }, @@ -437,9 +438,22 @@ "source": [ "display(vcps_back)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96b5c030-3087-402b-b33b-6daf6d6e6214", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -450,7 +464,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.11.6" } }, "nbformat": 4, From e33c099f87f345dc4ce1648e7e535a72116a4d85 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 10:06:29 -0500 Subject: [PATCH 07/10] applying black check --- tests/io/test_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index 66697cc7..7c151b95 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -618,7 +618,7 @@ def test_open_iris_datatree(iris0_file): ranges = [664] * 10 i = 0 for grp in dtree.groups: - if grp.startswith('/sweep_'): + if grp.startswith("/sweep_"): ds = dtree[grp].ds assert dict(ds.sizes) == {"azimuth": azimuths[i], "range": ranges[i]} assert set(ds.data_vars) & ( From e354a1e7061af3831a60233435d389ce9c32aba1 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 10:10:09 -0500 Subject: [PATCH 08/10] reformatting using ruff --- xradar/io/backends/iris.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xradar/io/backends/iris.py b/xradar/io/backends/iris.py index 81b83035..f5e78ad8 100644 --- a/xradar/io/backends/iris.py +++ b/xradar/io/backends/iris.py @@ -67,7 +67,7 @@ required_root_vars, sweep_vars_mapping, ) -from .common import _attach_sweep_groups, _assign_root +from .common import _assign_root, _attach_sweep_groups #: mapping from IRIS names to CfRadial2/ODIM iris_mapping = { From d263f3fdfd9c31fd0363307805dbef18611bec17 Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 10:41:42 -0500 Subject: [PATCH 09/10] adding Alfonso as colaborator --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index b0d3ad4c..0924c3bf 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -10,3 +10,4 @@ * Edouard Goudenhoofdt * Hamid Ali Syed +* Alfonso Ladino From 4fba5e92907d9fc881897f0482d6e09d67087f7a Mon Sep 17 00:00:00 2001 From: aladinor Date: Fri, 29 Mar 2024 10:42:13 -0500 Subject: [PATCH 10/10] updating history.md file --- docs/history.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/history.md b/docs/history.md index c9b16a44..5cb73dd4 100644 --- a/docs/history.md +++ b/docs/history.md @@ -2,6 +2,8 @@ ## Development Version (unreleased) +* ENH: Adding global variables and attributes to iris datatree ({pull}`166`) by [@aladinor](https://github.com/aladinor) + ## 0.5.0 (2024-03-28) * MNT: Update GitHub actions, address DeprecationWarnings ({pull}`153`) by [@kmuehlbauer](https://github.com/kmuehlbauer).