diff --git a/AUTHORS.md b/AUTHORS.md index 5f94be850c..733e9d572d 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -66,6 +66,7 @@ The following people have made contributions to this project: - [Lars Ørum Rasmussen (loerum)](https://github.com/loerum) - [Martin Raspaud (mraspaud)](https://github.com/mraspaud) - [William Roberts (wroberts4)](https://github.com/wroberts4) +- [Benjamin Rösner (BENR0)](https://github.com/BENR0) - [Pascale Roquet (roquetp)](https://github.com/roquetp) - [Kristian Rune Larsen](https://github.com/) - [RutgerK (RutgerK)](https://github.com/RutgerK) diff --git a/doc/source/_static/main.js b/doc/source/_static/main.js index f2da99d4e7..3197ce084e 100644 --- a/doc/source/_static/main.js +++ b/doc/source/_static/main.js @@ -1,10 +1,12 @@ $(document).ready( function () { $('table.datatable').DataTable( { - "paging": false, + "paging": true, + "pageLength": 15, "layout": { 'topStart': 'info', 'topEnd': 'search', - 'bottomStart': null + 'bottomStart': null, + 'bottomEnd': 'paging' }, "order": [[0, 'asc']] } ); diff --git a/doc/source/index.rst b/doc/source/index.rst index b229c904ee..66a069fcda 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,11 +7,11 @@ remote-sensing earth-observing satellite instruments. Satpy provides users with readers that convert geophysical parameters from various file formats to the common Xarray :class:`~xarray.DataArray` and :class:`~xarray.Dataset` classes for easier interoperability with other -scientific python libraries. Satpy also provides interfaces for creating -RGB (Red/Green/Blue) images and other composite types by combining data -from multiple instrument bands or products. Various atmospheric corrections -and visual enhancements are provided for improving the usefulness and quality -of output images. Output data can be written to +scientific python libraries. For a full list of available readers see :ref:`reader_table`. +Satpy also provides interfaces for creating RGB (Red/Green/Blue) images and other +composite types by combining data from multiple instrument bands or products. +Various atmospheric corrections and visual enhancements are provided for +improving the usefulness and quality of output images. Output data can be written to multiple output file formats such as PNG, GeoTIFF, and CF standard NetCDF files. Satpy also allows users to resample data to geographic projected grids (areas). Satpy is maintained by the open source @@ -80,34 +80,11 @@ Documentation Release Notes Security Policy -.. _reader_table: -.. include:: reader_table.rst - -.. _Status Description: .. note:: - Status description: - - Defunct - Most likely the reader is not functional. If it is there is a good chance of - bugs and/or performance problems (e.g. not ported to dask/xarray yet). Future - development is unclear. Users are encouraged to contribute (see section - :doc:`dev_guide/CONTRIBUTING` and/or get help on Slack or by opening a Github issue). - - Alpha - This denotes early development status. Reader is functional and implements some - or all of the nominal features. There might be bugs. Exactness of results is - not be guaranteed. Use at your own risk. - - Beta - This denotes final developement status. Reader is functional and implements all - nominal features. Results should be dependable but there might be bugs. Users - are actively encouraged to test and report bugs. - - Nominal - This denotes a finished status. Reader is functional and most likely no new - features will be introduced. It has been tested and there are no known bugs. + Please note that the reader table that used to be placed here has moved to the "reading" + section here: :ref:`reader_table`. Indices and tables ================== diff --git a/doc/source/reading.rst b/doc/source/reading.rst index b7264eeb6e..0dbea74046 100644 --- a/doc/source/reading.rst +++ b/doc/source/reading.rst @@ -23,6 +23,124 @@ To return additional reader information use `available_readers(as_dict=True)`:: >>> from satpy import available_readers >>> available_readers() + +.. _reader_table: + +Reader Table +------------ + +.. include:: reader_table.rst + +.. _Status Description: +.. note:: + + Status description: + + Defunct + Most likely the reader is not functional. If it is there is a good chance of + bugs and/or performance problems (e.g. not ported to dask/xarray yet). Future + development is unclear. Users are encouraged to contribute (see section + :doc:`dev_guide/CONTRIBUTING` and/or get help on Slack or by opening a Github issue). + + Alpha + This denotes early development status. Reader is functional and implements some + or all of the nominal features. There might be bugs. Exactness of results is + not be guaranteed. Use at your own risk. + + Beta + This denotes final developement status. Reader is functional and implements all + nominal features. Results should be dependable but there might be bugs. Users + are actively encouraged to test and report bugs. + + Nominal + This denotes a finished status. Reader is functional and most likely no new + features will be introduced. It has been tested and there are no known bugs. + +Documentation for specific readers +---------------------------------- + +SEVIRI L1.5 data readers +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.seviri_base + :noindex: + +SEVIRI HRIT format reader +""""""""""""""""""""""""" + +.. automodule:: satpy.readers.seviri_l1b_hrit + :noindex: + +SEVIRI Native format reader +""""""""""""""""""""""""""" + +.. automodule:: satpy.readers.seviri_l1b_native + :noindex: + +SEVIRI netCDF format reader +""""""""""""""""""""""""""" + +.. automodule:: satpy.readers.seviri_l1b_nc + :noindex: + + +Other xRIT-based readers +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.hrit_base + :noindex: + + +JMA HRIT format reader +^^^^^^^^^^^^^^^^^^^^^^ + + +.. automodule:: satpy.readers.hrit_jma + :noindex: + +GOES HRIT format reader +^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.goes_imager_hrit + :noindex: + +Electro-L HRIT format reader +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.electrol_hrit + :noindex: + +hdf-eos based readers +^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.modis_l1b + :noindex: + +.. automodule:: satpy.readers.modis_l2 + :noindex: + +satpy cf nc readers +^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.satpy_cf_nc + :noindex: + +hdf5 based readers +^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.agri_l1 + :noindex: + +.. automodule:: satpy.readers.ghi_l1 + :noindex: + +Arctica-M N1 HDF5 format reader +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. automodule:: satpy.readers.msu_gsa_l1b + :noindex: + + Filter loaded files =================== @@ -290,87 +408,3 @@ Adding a Reader to Satpy ======================== This is described in the developer guide, see :doc:`dev_guide/custom_reader`. - -Implemented readers -=================== - -SEVIRI L1.5 data readers ------------------------- - -.. automodule:: satpy.readers.seviri_base - :noindex: - -SEVIRI HRIT format reader -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.seviri_l1b_hrit - :noindex: - -SEVIRI Native format reader -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.seviri_l1b_native - :noindex: - -SEVIRI netCDF format reader -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.seviri_l1b_nc - :noindex: - - -Other xRIT-based readers ------------------------- - -.. automodule:: satpy.readers.hrit_base - :noindex: - - -JMA HRIT format reader -^^^^^^^^^^^^^^^^^^^^^^ - - -.. automodule:: satpy.readers.hrit_jma - :noindex: - -GOES HRIT format reader -^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.goes_imager_hrit - :noindex: - -Electro-L HRIT format reader -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.electrol_hrit - :noindex: - -hdf-eos based readers ---------------------- - -.. automodule:: satpy.readers.modis_l1b - :noindex: - -.. automodule:: satpy.readers.modis_l2 - :noindex: - -satpy cf nc readers ---------------------- - -.. automodule:: satpy.readers.satpy_cf_nc - :noindex: - -hdf5 based readers ------------------- - -.. automodule:: satpy.readers.agri_l1 - :noindex: - -.. automodule:: satpy.readers.ghi_l1 - :noindex: - -Arctica-M N1 HDF5 format reader -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. automodule:: satpy.readers.msu_gsa_l1b - :noindex: diff --git a/pyproject.toml b/pyproject.toml index c2ae1a890d..673405cfb1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,6 +41,7 @@ avhrr_l1b_gaclac = ["pygac >= 1.3.0"] modis_l1b = ["pyhdf", "python-geotiepoints >= 1.1.7"] geocat = ["pyhdf"] goci2 = ["netCDF4 >= 1.1.8"] +generic_image = ["rasterio", "rioxarray"] acspo = ["netCDF4 >= 1.1.8"] clavrx = ["netCDF4 >= 1.1.8"] viirs_l1b = ["netCDF4 >= 1.1.8"] diff --git a/satpy/readers/generic_image.py b/satpy/readers/generic_image.py index f6c983e8d5..5032b7bbb8 100644 --- a/satpy/readers/generic_image.py +++ b/satpy/readers/generic_image.py @@ -31,6 +31,7 @@ import dask.array as da import numpy as np import rasterio +import rioxarray # noqa: F401 # need by xarray with the engine rasterio import xarray as xr from pyresample import utils @@ -76,6 +77,9 @@ def read(self): if hasattr(dataset, "crs") and dataset.crs is not None: self.area = utils.get_area_def_from_raster(dataset) + # xarray use the engine 'rasterio' to open the file, but + # its actually rioxarray used in the backend. + # however, error is not explicit enough (see https://github.com/pydata/xarray/issues/7831) data = xr.open_dataset(self.finfo["filename"], engine="rasterio", chunks={"band": 1, "y": CHUNK_SIZE, "x": CHUNK_SIZE}, mask_and_scale=False)["band_data"] if hasattr(dataset, "nodatavals"): diff --git a/satpy/readers/hdf5_utils.py b/satpy/readers/hdf5_utils.py index cfce968cf3..c08b624abf 100644 --- a/satpy/readers/hdf5_utils.py +++ b/satpy/readers/hdf5_utils.py @@ -18,19 +18,21 @@ """Helpers for reading hdf5-based files.""" import logging +import os import dask.array as da +import dask.config as dc import h5py import numpy as np import xarray as xr +from dask.array.core import normalize_chunks +from dask.base import tokenize from satpy.readers import open_file_or_filename from satpy.readers.file_handlers import BaseFileHandler from satpy.readers.utils import np2str -from satpy.utils import get_legacy_chunk_size LOG = logging.getLogger(__name__) -CHUNK_SIZE = get_legacy_chunk_size() class HDF5FileHandler(BaseFileHandler): @@ -102,7 +104,7 @@ def __getitem__(self, key): # these datasets are closed and inaccessible when the file is closed, need to reopen f_obj = open_file_or_filename(self.filename) dset = h5py.File(f_obj, "r")[key] - dset_data = da.from_array(dset, chunks=CHUNK_SIZE) + dset_data = from_h5_array(dset) attrs = self._attrs_cache.get(key, dset.attrs) if dset.ndim == 2: return xr.DataArray(dset_data, dims=["y", "x"], attrs=attrs) @@ -120,3 +122,14 @@ def get(self, item, default=None): return self[item] else: return default + + +def from_h5_array(h5dset): + """Create a dask array from an h5py dataset, ensuring uniqueness of the dask array name.""" + chunk_size = dc.get("array.chunk-size") + + chunks = normalize_chunks(chunk_size, dtype=h5dset.dtype, previous_chunks=h5dset.chunks, shape=h5dset.shape) + name = h5dset.name + "-" + tokenize(os.fspath(h5dset.file.filename), h5dset.name, chunks) + + dset_data = da.from_array(h5dset, chunks=chunks, name=name) + return dset_data diff --git a/satpy/readers/sar_c_safe.py b/satpy/readers/sar_c_safe.py index a5ec535462..f04656bfe9 100644 --- a/satpy/readers/sar_c_safe.py +++ b/satpy/readers/sar_c_safe.py @@ -46,6 +46,7 @@ import defusedxml.ElementTree as ET import numpy as np import rasterio +import rioxarray # noqa F401 # xarray open_dataset use engine rasterio, which use rioxarray import xarray as xr from dask import array as da from geotiepoints.geointerpolator import lonlat2xyz, xyz2lonlat diff --git a/satpy/tests/reader_tests/test_hdf5_utils.py b/satpy/tests/reader_tests/test_hdf5_utils.py index 4882701c94..8a7d3ef52d 100644 --- a/satpy/tests/reader_tests/test_hdf5_utils.py +++ b/satpy/tests/reader_tests/test_hdf5_utils.py @@ -150,3 +150,13 @@ def test_all_basic(self): assert "fake_ds" not in file_handler assert isinstance(file_handler["ds2_f/attr/test_ref"], np.ndarray) + + def test_array_name_uniqueness(self): + """Test the dask array generated from an hdf5 dataset stay constant and unique.""" + from satpy.readers.hdf5_utils import HDF5FileHandler + file_handler = HDF5FileHandler("test.h5", {}, {}) + + dsname = "test_group/ds1_f" + + assert file_handler[dsname].data.name == file_handler[dsname].data.name + assert file_handler[dsname].data.name.startswith("/" + dsname)