Merge branch 'main' into shared_fci_netcdf_metadata

AUTHORS.md

@@ -66,6 +66,7 @@ The following people have made contributions to this project:
 - [Lars Ørum Rasmussen (loerum)](https://github.com/loerum)
 - [Martin Raspaud (mraspaud)](https://github.com/mraspaud)
 - [William Roberts (wroberts4)](https://github.com/wroberts4)
+- [Benjamin Rösner (BENR0)](https://github.com/BENR0)
 - [Pascale Roquet (roquetp)](https://github.com/roquetp)
 - [Kristian Rune Larsen](https://github.com/)
 - [RutgerK (RutgerK)](https://github.com/RutgerK)

doc/source/_static/main.js

@@ -1,10 +1,12 @@
 $(document).ready( function () {
     $('table.datatable').DataTable( {
-    "paging": false,
+    "paging": true,
+    "pageLength": 15,
     "layout": {
         'topStart': 'info',
         'topEnd': 'search',
-        'bottomStart': null
+        'bottomStart': null,
+        'bottomEnd': 'paging'
     },
     "order": [[0, 'asc']]
 } );

doc/source/index.rst

@@ -7,11 +7,11 @@ remote-sensing earth-observing satellite instruments. Satpy
 provides users with readers that convert geophysical parameters from various
 file formats to the common Xarray :class:`~xarray.DataArray` and
 :class:`~xarray.Dataset` classes for easier interoperability with other
-scientific python libraries. Satpy also provides interfaces for creating
-RGB (Red/Green/Blue) images and other composite types by combining data
-from multiple instrument bands or products. Various atmospheric corrections
-and visual enhancements are provided for improving the usefulness and quality
-of output images. Output data can be written to
+scientific python libraries. For a full list of available readers see :ref:`reader_table`.
+Satpy also provides interfaces for creating RGB (Red/Green/Blue) images and other
+composite types by combining data from multiple instrument bands or products.
+Various atmospheric corrections and visual enhancements are provided for
+improving the usefulness and quality of output images. Output data can be written to
 multiple output file formats such as PNG, GeoTIFF, and CF standard NetCDF
 files. Satpy also allows users to resample data to geographic projected grids
 (areas). Satpy is maintained by the open source
@@ -80,34 +80,11 @@ Documentation
     Release Notes <https://github.com/pytroll/satpy/blob/main/CHANGELOG.md>
     Security Policy <https://github.com/pytroll/satpy/blob/main/SECURITY.md>
 
-.. _reader_table:
 
-.. include:: reader_table.rst
-
-.. _Status Description:
 .. note::
 
-    Status description:
-
-    Defunct
-        Most likely the reader is not functional. If it is there is a good chance of
-        bugs and/or performance problems (e.g. not ported to dask/xarray yet). Future
-        development is unclear. Users are encouraged to contribute (see section
-        :doc:`dev_guide/CONTRIBUTING` and/or get help on Slack or by opening a Github issue).
-
-    Alpha
-        This denotes early development status. Reader is functional and implements some
-        or all of the nominal features. There might be bugs. Exactness of results is
-        not be guaranteed. Use at your own risk.
-
-    Beta
-        This denotes final developement status. Reader is functional and implements all
-        nominal features. Results should be dependable but there might be bugs. Users
-        are actively encouraged to test and report bugs.
-
-    Nominal
-        This denotes a finished status. Reader is functional and most likely no new
-        features will be introduced. It has been tested and there are no known bugs.
+    Please note that the reader table that used to be placed here has moved to the "reading"
+    section here:  :ref:`reader_table`.
 
 Indices and tables
 ==================

doc/source/reading.rst

@@ -23,6 +23,124 @@ To return additional reader information use `available_readers(as_dict=True)`::
     >>> from satpy import available_readers
     >>> available_readers()
 
+
+.. _reader_table:
+
+Reader Table
+------------
+
+.. include:: reader_table.rst
+
+.. _Status Description:
+.. note::
+
+    Status description:
+
+    Defunct
+        Most likely the reader is not functional. If it is there is a good chance of
+        bugs and/or performance problems (e.g. not ported to dask/xarray yet). Future
+        development is unclear. Users are encouraged to contribute (see section
+        :doc:`dev_guide/CONTRIBUTING` and/or get help on Slack or by opening a Github issue).
+
+    Alpha
+        This denotes early development status. Reader is functional and implements some
+        or all of the nominal features. There might be bugs. Exactness of results is
+        not be guaranteed. Use at your own risk.
+
+    Beta
+        This denotes final developement status. Reader is functional and implements all
+        nominal features. Results should be dependable but there might be bugs. Users
+        are actively encouraged to test and report bugs.
+
+    Nominal
+        This denotes a finished status. Reader is functional and most likely no new
+        features will be introduced. It has been tested and there are no known bugs.
+
+Documentation for specific readers
+----------------------------------
+
+SEVIRI L1.5 data readers
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.seviri_base
+    :noindex:
+
+SEVIRI HRIT format reader
+"""""""""""""""""""""""""
+
+.. automodule:: satpy.readers.seviri_l1b_hrit
+    :noindex:
+
+SEVIRI Native format reader
+"""""""""""""""""""""""""""
+
+.. automodule:: satpy.readers.seviri_l1b_native
+    :noindex:
+
+SEVIRI netCDF format reader
+"""""""""""""""""""""""""""
+
+.. automodule:: satpy.readers.seviri_l1b_nc
+    :noindex:
+
+
+Other xRIT-based readers
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.hrit_base
+    :noindex:
+
+
+JMA HRIT format reader
+^^^^^^^^^^^^^^^^^^^^^^
+
+
+.. automodule:: satpy.readers.hrit_jma
+    :noindex:
+
+GOES HRIT format reader
+^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.goes_imager_hrit
+    :noindex:
+
+Electro-L HRIT format reader
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.electrol_hrit
+    :noindex:
+
+hdf-eos based readers
+^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.modis_l1b
+    :noindex:
+
+.. automodule:: satpy.readers.modis_l2
+    :noindex:
+
+satpy cf nc readers
+^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.satpy_cf_nc
+    :noindex:
+
+hdf5 based readers
+^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.agri_l1
+    :noindex:
+
+.. automodule:: satpy.readers.ghi_l1
+    :noindex:
+
+Arctica-M N1 HDF5 format reader
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+.. automodule:: satpy.readers.msu_gsa_l1b
+    :noindex:
+
+
 Filter loaded files
 ===================
 
@@ -290,87 +408,3 @@ Adding a Reader to Satpy
 ========================
 
 This is described in the developer guide, see :doc:`dev_guide/custom_reader`.
-
-Implemented readers
-===================
-
-SEVIRI L1.5 data readers
-------------------------
-
-.. automodule:: satpy.readers.seviri_base
-    :noindex:
-
-SEVIRI HRIT format reader
-^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.seviri_l1b_hrit
-    :noindex:
-
-SEVIRI Native format reader
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.seviri_l1b_native
-    :noindex:
-
-SEVIRI netCDF format reader
-^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.seviri_l1b_nc
-    :noindex:
-
-
-Other xRIT-based readers
-------------------------
-
-.. automodule:: satpy.readers.hrit_base
-    :noindex:
-
-
-JMA HRIT format reader
-^^^^^^^^^^^^^^^^^^^^^^
-
-
-.. automodule:: satpy.readers.hrit_jma
-    :noindex:
-
-GOES HRIT format reader
-^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.goes_imager_hrit
-    :noindex:
-
-Electro-L HRIT format reader
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.electrol_hrit
-    :noindex:
-
-hdf-eos based readers
----------------------
-
-.. automodule:: satpy.readers.modis_l1b
-    :noindex:
-
-.. automodule:: satpy.readers.modis_l2
-    :noindex:
-
-satpy cf nc readers
----------------------
-
-.. automodule:: satpy.readers.satpy_cf_nc
-    :noindex:
-
-hdf5 based readers
-------------------
-
-.. automodule:: satpy.readers.agri_l1
-    :noindex:
-
-.. automodule:: satpy.readers.ghi_l1
-    :noindex:
-
-Arctica-M N1 HDF5 format reader
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-.. automodule:: satpy.readers.msu_gsa_l1b
-    :noindex:

pyproject.toml

@@ -41,6 +41,7 @@ avhrr_l1b_gaclac = ["pygac >= 1.3.0"]
 modis_l1b = ["pyhdf", "python-geotiepoints >= 1.1.7"]
 geocat = ["pyhdf"]
 goci2 = ["netCDF4 >= 1.1.8"]
+generic_image = ["rasterio", "rioxarray"]
 acspo = ["netCDF4 >= 1.1.8"]
 clavrx = ["netCDF4 >= 1.1.8"]
 viirs_l1b = ["netCDF4 >= 1.1.8"]

satpy/readers/generic_image.py

@@ -31,6 +31,7 @@
 import dask.array as da
 import numpy as np
 import rasterio
+import rioxarray  # noqa: F401  # need by xarray with the engine rasterio
 import xarray as xr
 from pyresample import utils
 
@@ -76,6 +77,9 @@ def read(self):
         if hasattr(dataset, "crs") and dataset.crs is not None:
             self.area = utils.get_area_def_from_raster(dataset)
 
+        # xarray use the engine 'rasterio' to open the file, but
+        #   its actually rioxarray used in the backend.
+        #   however, error is not explicit enough (see https://github.com/pydata/xarray/issues/7831)
         data = xr.open_dataset(self.finfo["filename"], engine="rasterio",
                                chunks={"band": 1, "y": CHUNK_SIZE, "x": CHUNK_SIZE}, mask_and_scale=False)["band_data"]
         if hasattr(dataset, "nodatavals"):

satpy/readers/hdf5_utils.py

@@ -18,19 +18,21 @@
 """Helpers for reading hdf5-based files."""
 
 import logging
+import os
 
 import dask.array as da
+import dask.config as dc
 import h5py
 import numpy as np
 import xarray as xr
+from dask.array.core import normalize_chunks
+from dask.base import tokenize
 
 from satpy.readers import open_file_or_filename
 from satpy.readers.file_handlers import BaseFileHandler
 from satpy.readers.utils import np2str
-from satpy.utils import get_legacy_chunk_size
 
 LOG = logging.getLogger(__name__)
-CHUNK_SIZE = get_legacy_chunk_size()
 
 
 class HDF5FileHandler(BaseFileHandler):
@@ -102,7 +104,7 @@ def __getitem__(self, key):
             # these datasets are closed and inaccessible when the file is closed, need to reopen
             f_obj = open_file_or_filename(self.filename)
             dset = h5py.File(f_obj, "r")[key]
-            dset_data = da.from_array(dset, chunks=CHUNK_SIZE)
+            dset_data = from_h5_array(dset)
             attrs = self._attrs_cache.get(key, dset.attrs)
             if dset.ndim == 2:
                 return xr.DataArray(dset_data, dims=["y", "x"], attrs=attrs)
@@ -120,3 +122,14 @@ def get(self, item, default=None):
             return self[item]
         else:
             return default
+
+
+def from_h5_array(h5dset):
+    """Create a dask array from an h5py dataset, ensuring uniqueness of the dask array name."""
+    chunk_size = dc.get("array.chunk-size")
+
+    chunks = normalize_chunks(chunk_size, dtype=h5dset.dtype, previous_chunks=h5dset.chunks, shape=h5dset.shape)
+    name = h5dset.name + "-" + tokenize(os.fspath(h5dset.file.filename), h5dset.name, chunks)
+
+    dset_data = da.from_array(h5dset, chunks=chunks, name=name)
+    return dset_data

satpy/readers/sar_c_safe.py

@@ -46,6 +46,7 @@
 import defusedxml.ElementTree as ET
 import numpy as np
 import rasterio
+import rioxarray  # noqa F401  # xarray open_dataset use engine rasterio, which use rioxarray
 import xarray as xr
 from dask import array as da
 from geotiepoints.geointerpolator import lonlat2xyz, xyz2lonlat

satpy/tests/reader_tests/test_hdf5_utils.py

@@ -150,3 +150,13 @@ def test_all_basic(self):
         assert "fake_ds" not in file_handler
 
         assert isinstance(file_handler["ds2_f/attr/test_ref"], np.ndarray)
+
+    def test_array_name_uniqueness(self):
+        """Test the dask array generated from an hdf5 dataset stay constant and unique."""
+        from satpy.readers.hdf5_utils import HDF5FileHandler
+        file_handler = HDF5FileHandler("test.h5", {}, {})
+
+        dsname = "test_group/ds1_f"
+
+        assert file_handler[dsname].data.name == file_handler[dsname].data.name
+        assert file_handler[dsname].data.name.startswith("/" + dsname)