Merge remote-tracking branch 'upstream/main' into main

.pre-commit-config.yaml

@@ -20,7 +20,7 @@ repos:
       - id: bandit
         args: [--ini, .bandit]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: 'v1.3.0'  # Use the sha / tag you want to point at
+    rev: 'v1.4.1'  # Use the sha / tag you want to point at
     hooks:
       - id: mypy
         additional_dependencies:

AUTHORS.md

@@ -27,6 +27,7 @@ The following people have made contributions to this project:
 - [Ulrik Egede (egede)](https://github.com/egede)
 - [Joleen Feltz (joleenf)](https://github.com/joleenf)
 - [Stephan Finkensieper (sfinkens)](https://github.com/sfinkens) - Deutscher Wetterdienst
+- [Gionata Ghiggi (ghiggi)](https://github.com/ghiggi)
 - [Andrea Grillini (AppLEaDaY)](https://github.com/AppLEaDaY)
 - [Blanka Gvozdikova (gvozdikb)](https://github.com/gvozdikb)
 - [Nina Håkansson (ninahakansson)](https://github.com/ninahakansson)

continuous_integration/environment.yaml

@@ -5,10 +5,12 @@ dependencies:
   - xarray!=2022.9.0
   - dask
   - distributed
+  - dask-image
   - donfig
   - appdirs
   - toolz
   - Cython
+  - numba
   - sphinx
   - cartopy
   - panel>=0.12.7

doc/rtd_environment.yml

@@ -6,11 +6,13 @@ dependencies:
   - pip
   - appdirs
   - dask
+  - dask-image
   - defusedxml
   - donfig
   # 2.19.1 seems to cause library linking issues
   - eccodes>=2.20
   - graphviz
+  - numba
   - numpy
   - pillow
   - pooch

doc/source/composites.rst

@@ -173,7 +173,7 @@ In the case below, the image shows its day portion and day/night
 transition with night portion blacked-out instead of transparent::
 
     >>> from satpy.composites import DayNightCompositor
-    >>> compositor = DayNightCompositor("dnc", lim_low=85., lim_high=88., day_night="day_only", need_alpha=False)
+    >>> compositor = DayNightCompositor("dnc", lim_low=85., lim_high=88., day_night="day_only", include_alpha=False)
     >>> composite = compositor([local_scene['true_color'])
 
 RealisticColors

doc/source/config.rst

@@ -258,6 +258,23 @@ as part of the :func:`~satpy.modifiers.angles.get_angles` and
 used by multiple modifiers and composites including the default rayleigh
 correction.
 
+Clipping Negative Infrared Radiances
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* **Environment variable**: ``SATPY_READERS__CLIP_NEGATIVE_RADIANCES``
+* **YAML/Config Key**: ``readers.clip_negative_radiances``
+* **Default**: False
+
+Whether to clip negative infrared radiances to the minimum allowable value before
+computing the brightness temperature.
+If ``clip_negative_radiances=False``, pixels with negative radiances will have
+``np.nan`` brightness temperatures.
+
+Clipping of negative radiances is currently implemented for the following readers:
+
+* ``abi_l1b``
+
+
 Temporary Directory
 ^^^^^^^^^^^^^^^^^^^
 

satpy/_config.py

@@ -62,6 +62,9 @@ def impr_files(module_name: str) -> Path:
     'demo_data_dir': '.',
     'download_aux': True,
     'sensor_angles_position_preference': 'actual',
+    'readers': {
+        'clip_negative_radiances': False,
+    },
 }
 
 # Satpy main configuration object

satpy/_scene_converters.py

@@ -0,0 +1,124 @@
+# Copyright (c) 2023 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Helper functions for converting the Scene object to some other object."""
+
+import xarray as xr
+
+from satpy.dataset import DataID
+
+
+def _get_dataarrays_from_identifiers(scn, identifiers):
+    """Return a list of DataArray based on a single or list of identifiers.
+
+    An identifier can be a DataID or a string with name of a valid DataID.
+    """
+    if isinstance(identifiers, (str, DataID)):
+        identifiers = [identifiers]
+
+    if identifiers is not None:
+        dataarrays = [scn[ds] for ds in identifiers]
+    else:
+        dataarrays = [scn._datasets.get(ds) for ds in scn._wishlist]
+        dataarrays = [dataarray for dataarray in dataarrays if dataarray is not None]
+    return dataarrays
+
+
+def to_xarray(scn,
+              datasets=None,  # DataID
+              header_attrs=None,
+              exclude_attrs=None,
+              flatten_attrs=False,
+              pretty=True,
+              include_lonlats=True,
+              epoch=None,
+              include_orig_name=True,
+              numeric_name_prefix='CHANNEL_'):
+    """Merge all xr.DataArray(s) of a satpy.Scene to a CF-compliant xarray object.
+
+    If all Scene DataArrays are on the same area, it returns an xr.Dataset.
+    If Scene DataArrays are on different areas, currently it fails, although
+    in future we might return a DataTree object, grouped by area.
+
+    Parameters
+    ----------
+    scn: satpy.Scene
+        Satpy Scene.
+    datasets (iterable):
+        List of Satpy Scene datasets to include in the output xr.Dataset.
+        Elements can be string name, a wavelength as a number, a DataID,
+        or DataQuery object.
+        If None (the default), it include all loaded Scene datasets.
+    header_attrs:
+        Global attributes of the output xr.Dataset.
+    epoch (str):
+        Reference time for encoding the time coordinates (if available).
+        Example format: "seconds since 1970-01-01 00:00:00".
+        If None, the default reference time is retrieved using "from satpy.cf_writer import EPOCH"
+    flatten_attrs (bool):
+        If True, flatten dict-type attributes.
+    exclude_attrs (list):
+        List of xr.DataArray attribute names to be excluded.
+    include_lonlats (bool):
+        If True, it includes 'latitude' and 'longitude' coordinates.
+        If the 'area' attribute is a SwathDefinition, it always includes
+        latitude and longitude coordinates.
+    pretty (bool):
+        Don't modify coordinate names, if possible. Makes the file prettier,
+        but possibly less consistent.
+    include_orig_name (bool).
+        Include the original dataset name as a variable attribute in the xr.Dataset.
+    numeric_name_prefix (str):
+        Prefix to add the each variable with name starting with a digit.
+        Use '' or None to leave this out.
+
+    Returns
+    -------
+    ds, xr.Dataset
+        A CF-compliant xr.Dataset
+
+    """
+    from satpy.writers.cf_writer import EPOCH, collect_cf_datasets
+
+    if epoch is None:
+        epoch = EPOCH
+
+    # Get list of DataArrays
+    if datasets is None:
+        datasets = list(scn.keys())  # list all loaded DataIDs
+    list_dataarrays = _get_dataarrays_from_identifiers(scn, datasets)
+
+    # Check that some DataArray could be returned
+    if len(list_dataarrays) == 0:
+        return xr.Dataset()
+
+    # Collect xr.Dataset for each group
+    grouped_datasets, header_attrs = collect_cf_datasets(list_dataarrays=list_dataarrays,
+                                                         header_attrs=header_attrs,
+                                                         exclude_attrs=exclude_attrs,
+                                                         flatten_attrs=flatten_attrs,
+                                                         pretty=pretty,
+                                                         include_lonlats=include_lonlats,
+                                                         epoch=epoch,
+                                                         include_orig_name=include_orig_name,
+                                                         numeric_name_prefix=numeric_name_prefix,
+                                                         groups=None)
+    if len(grouped_datasets) == 1:
+        ds = grouped_datasets[None]
+        return ds
+    else:
+        msg = """The Scene object contains datasets with different areas.
+                  Resample the Scene to have matching dimensions using i.e. scn.resample(resampler="native") """
+        raise NotImplementedError(msg)

satpy/etc/composites/visir.yaml

@@ -96,6 +96,11 @@ modifiers:
     - solar_azimuth_angle
     - solar_zenith_angle
 
+  median5x5:
+    modifier: !!python/name:satpy.modifiers.filters.Median
+    median_filter_params:
+      size: 5
+
 composites:
 
   airmass:

satpy/etc/readers/gms5-vissr_l1b.yaml

@@ -0,0 +1,99 @@
+reader:
+  name: gms5-vissr_l1b
+  short_name: GMS-5 VISSR L1b
+  long_name: GMS-5 VISSR Level 1b
+  description: >
+    Reader for GMS-5 VISSR Level 1b data. References:
+
+    - https://www.data.jma.go.jp/mscweb/en/operation/fig/VISSR_FORMAT_GMS-5.pdf
+    - https://www.data.jma.go.jp/mscweb/en/operation/fig/GMS_Users_Guide_3rd_Edition_Rev1.pdf
+
+  status: Alpha
+  supports_fsspec: true
+  sensors: [gms5-vissr]
+  default_channels: []
+  reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
+
+file_types:
+    gms5_vissr_vis:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_VIS.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_VIS.{mode}.IMG.gz'
+
+    gms5_vissr_ir1:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR1.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR1.{mode}.IMG.gz'
+
+    gms5_vissr_ir2:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR2.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR2.{mode}.IMG.gz'
+
+
+    gms5_vissr_ir3:
+        file_reader: !!python/name:satpy.readers.gms.gms5_vissr_l1b.GMS5VISSRFileHandler
+        file_patterns:
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR3.{mode}.IMG'
+        - 'VISSR_{start_time:%Y%m%d_%H%M}_IR3.{mode}.IMG.gz'
+
+
+datasets:
+  VIS:
+    name: VIS
+    sensor: gms5-vissr
+    wavelength: [0.55, 0.73, 0.9]
+    resolution: 1250
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      reflectance:
+        standard_name: toa_bidirectional_reflectance
+        units: "%"
+    file_type: gms5_vissr_vis
+
+  IR1:
+    name: IR1
+    sensor: gms5-vissr
+    wavelength: [10.5, 11.0, 11.5]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir1
+
+  IR2:
+    name: IR2
+    sensor: gms5-vissr
+    wavelength: [11.5, 12.0, 12.5]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir2
+
+  IR3:
+    name: IR3
+    sensor: gms5-vissr
+    wavelength: [6.5, 6.75, 7.0]
+    resolution: 5000
+    calibration:
+      counts:
+        standard_name: counts
+        units: 1
+      brightness_temperature:
+        standard_name: toa_brightness_temperature
+        units: K
+    file_type: gms5_vissr_ir3

satpy/etc/readers/nwcsaf-pps_nc.yaml

@@ -350,4 +350,4 @@ datasets:
     name: cmic_dcot
     file_key: dcot
     file_type: [nc_nwcsaf_cpp, nc_nwcsaf_cmic]
-    coordinates: [lon, lat]
+    coordinates: [lon, lat]

satpy/modifiers/filters.py

@@ -0,0 +1,34 @@
+"""Tests for image filters."""
+import logging
+
+import xarray as xr
+
+from satpy.modifiers import ModifierBase
+
+logger = logging.getLogger(__name__)
+
+
+class Median(ModifierBase):
+    """Apply a median filter to the band."""
+
+    def __init__(self, median_filter_params, **kwargs):
+        """Create the instance.
+
+        Args:
+            median_filter_params: The arguments to pass to dask-image's median_filter function. For example, {size: 3}
+                                  makes give the median filter a kernel of size 3.
+
+        """
+        self.median_filter_params = median_filter_params
+        super().__init__(**kwargs)
+
+    def __call__(self, arrays, **info):
+        """Get the median filtered band."""
+        from dask_image.ndfilters import median_filter
+
+        data = arrays[0]
+        logger.debug(f"Apply median filtering with parameters {self.median_filter_params}.")
+        res = xr.DataArray(median_filter(data.data, **self.median_filter_params),
+                           dims=data.dims, attrs=data.attrs, coords=data.coords)
+        self.apply_modifier_info(data, res)
+        return res