Merge branch 'main' of github.com:pytroll/satpy into clavrx_cloud_rgb

asv.conf.json

@@ -46,7 +46,7 @@
     // determined by looking for tools on the PATH environment
     // variable.
     //"environment_type": "virtualenv",
-    "environment_type": "conda",
+    "environment_type": "mamba",
 
     // timeout in seconds for installing any dependencies in environment
     // defaults to 10 min
@@ -58,7 +58,7 @@
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
     // "pythons": ["2.7", "3.6"],
-    "pythons": ["3.9", "3.10"],
+    "pythons": ["3.11", "3.12"],
 
     // The list of conda channel names to be searched for benchmark
     // dependency packages in the specified order
@@ -80,14 +80,14 @@
     //     "pip+emcee": [""],   // emcee is only available for install with pip.
     // },
     "matrix": {
-        "pyresample": ["1.22.3"],
-        "trollimage": ["1.17.0"],
-        "pyorbital": ["1.7.1"],
-        "pyspectral": ["0.10.6"],
-        "rasterio": ["1.2.10"],
-        "dask": ["2021.12.0"],
-        "xarray": ["0.20.2"],
-        "numpy": ["1.22.0"],
+        "pyresample": ["1.27.1"],
+        "trollimage": ["1.22.2"],
+        "pyorbital": ["1.8.1"],
+        "pyspectral": ["0.13.0"],
+        "rasterio": ["1.3.9"],
+        "dask": ["2024.1.1"],
+        "xarray": ["2024.1.1"],
+        "numpy": ["1.26.0"],
         "s3fs": [],
         "h5py": [],
         "netCDF4": [],

continuous_integration/environment.yaml

@@ -43,7 +43,7 @@ dependencies:
   - python-eccodes
   # 2.19.1 seems to cause library linking issues
   - eccodes>=2.20
-  - pytest
+  - pytest<8.0.0
   - pytest-cov
   - pytest-lazy-fixture
   - fsspec

satpy/etc/readers/nwcsaf-geo.yaml

@@ -77,6 +77,11 @@ datasets:
     resolution: 3000
     file_type: nc_nwcsaf_cma
 
+  cma_quality:
+    name: cma_quality
+    resolution: 3000
+    file_type: nc_nwcsaf_cma
+
   cma_pal:
     name: cma_pal
     resolution: 3000
@@ -129,6 +134,11 @@ datasets:
     resolution: 3000
     file_type: nc_nwcsaf_ct
 
+  ct_status_flag:
+    name: ct_status_flag
+    resolution: 3000
+    file_type: nc_nwcsaf_ct
+
   ct_pal:
     name: ct_pal
     resolution: 3000
@@ -370,7 +380,7 @@ datasets:
     file_type: nc_nwcsaf_crr-ph
 
   crrph_pal:
-    name: crrph_intensity_pal
+    name: crrph_pal
     resolution: 3000
     file_type: nc_nwcsaf_crr-ph
 
@@ -390,7 +400,7 @@ datasets:
     file_type: nc_nwcsaf_crr-ph
 
   crrph_status_flag:
-    name: crrph_status
+    name: crrph_status_flag
     resolution: 3000
     file_type: nc_nwcsaf_crr-ph
 
@@ -689,18 +699,38 @@ datasets:
     file_type: nc_nwcsaf_rdt
 
 # ----ASII products in multiple files ------------
+  # until v2018
   asii_turb_trop_prob:
     name: asii_turb_trop_prob
     resolution: 3000
     file_type: [nc_nwcsaf_asii_tf, nc_nwcsaf_asii]
 
+  # until v2018
   asii_turb_prob_pal:
     name: asii_turb_prob_pal
     resolution: 3000
     file_type: [nc_nwcsaf_asii_tf, nc_nwcsaf_asii_gw]
 
 # ----ASII-TF product ------------
 
+  # v2021 onwards
+  asiitf_prob:
+    name: asiitf_prob
+    resolution: 3000
+    file_type: nc_nwcsaf_asii_tf
+
+  # v2021 onwards
+  asiitf_prob_pal:
+    name: asiitf_prob_pal
+    file_type: nc_nwcsaf_asii_tf
+
+  # v2021 onwards
+  asiitf_status_flag:
+    name: asiitf_status_flag
+    resolution: 3000
+    file_type: nc_nwcsaf_asii_tf
+
+  # until v2018
   asii_turb_prob_status_flag:
     name: asii_turb_trop_prob_status_flag
     resolution: 3000
@@ -718,11 +748,30 @@ datasets:
 
 # ----ASII-GW product ------------
 
+  # v2021 onwards
+  asiigw_wv_prob:
+    name: asiigw_wv_prob
+    resolution: 3000
+    file_type: nc_nwcsaf_asii_gw
+
+  # v2021 onwards
+  asiigw_status_flag:
+    name: asiigw_status_flag
+    resolution: 3000
+    file_type: nc_nwcsaf_asii_gw
+
+  # v2021 onwards
+  asiigw_wv_prob_pal:
+    name: asiigw_wv_prob_pal
+    file_type: nc_nwcsaf_asii_gw
+
+  # until v2018
   asii_turb_wave_prob:
     name: asii_turb_wave_prob
     resolution: 3000
     file_type: nc_nwcsaf_asii_gw
 
+  # until v2018
   asii_turb_wave_prob_status_flag:
     name: asii_turb_wave_prob_status_flag
     resolution: 3000

satpy/readers/fy4_base.py

@@ -28,6 +28,7 @@
 
 import dask.array as da
 import numpy as np
+import numpy.typing as npt
 import xarray as xr
 
 from satpy._compat import cached_property
@@ -86,7 +87,7 @@ def scale(dn, slope, offset):
 
         return ref
 
-    def apply_lut(self, data, lut):
+    def _apply_lut(self, data: xr.DataArray, lut: npt.NDArray[np.float32]) -> xr.DataArray:
         """Calibrate digital number (DN) by applying a LUT.
 
         Args:
@@ -96,8 +97,16 @@ def apply_lut(self, data, lut):
             Calibrated quantity
         """
         # append nan to the end of lut for fillvalue
+        fill_value = data.attrs.get("FillValue")
+        if fill_value is not None and fill_value.item() <= lut.shape[0] - 1:
+            # If LUT includes the fill_value, remove that entry and everything
+            # after it.
+            # Ex. C07 has a LUT of 65536 elements, but fill value is 65535
+            # This is considered a bug in the input file format
+            lut = lut[:fill_value.item()]
+
         lut = np.append(lut, np.nan)
-        data.data = da.where(data.data > lut.shape[0], lut.shape[0] - 1, data.data)
+        data.data = da.where(data.data >= lut.shape[0], lut.shape[0] - 1, data.data)
         res = data.data.map_blocks(self._getitem, lut, dtype=lut.dtype)
         res = xr.DataArray(res, dims=data.dims,
                            attrs=data.attrs, coords=data.coords)
@@ -138,8 +147,8 @@ def calibrate(self, data, ds_info, ds_name, file_key):
             raise NotImplementedError("Calibration to radiance is not supported.")
         # Apply range limits, but not for counts or we convert to float!
         if calibration != "counts":
-            data = data.where((data >= min(data.attrs["valid_range"])) &
-                              (data <= max(data.attrs["valid_range"])))
+            data = data.where((data >= min(ds_info["valid_range"])) &
+                              (data <= max(ds_info["valid_range"])))
         else:
             data.attrs["_FillValue"] = data.attrs["FillValue"].item()
         return data
@@ -182,7 +191,7 @@ def calibrate_to_bt(self, data, ds_info, ds_name):
             lut = self[lut_key]
 
         # the value of dn is the index of brightness_temperature
-        data = self.apply_lut(data, lut)
+        data = self._apply_lut(data, lut.compute().data)
         ds_info["valid_range"] = lut.attrs["valid_range"]
         return data