From 85944b5d314d6aac80a9d154cb1de559943a648a Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Wed, 4 Dec 2024 15:36:25 +0100 Subject: [PATCH 01/16] Clean up --- satpy/readers/hrit_base.py | 6 ++---- satpy/readers/seviri_l1b_hrit.py | 12 ++++++------ satpy/readers/utils.py | 8 ++++---- .../tests/reader_tests/test_seviri_l1b_hrit.py | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 665fb1b7c7..492ba3be06 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -36,7 +36,6 @@ from pyresample import geometry import satpy.readers.utils as utils -from satpy.readers import FSFile from satpy.readers.eum_base import time_cds_short from satpy.readers.file_handlers import BaseFileHandler from satpy.readers.seviri_base import dec10216 @@ -117,8 +116,7 @@ class HRITFileHandler(BaseFileHandler): def __init__(self, filename, filename_info, filetype_info, hdr_info): """Initialize the reader.""" - super(HRITFileHandler, self).__init__(filename, filename_info, - filetype_info) + super().__init__(filename, filename_info, filetype_info) self.mda = {} self.hdr_info = hdr_info @@ -318,7 +316,7 @@ def _read_data_from_file(self): return self._read_data_from_disk() def _is_file_like(self): - return isinstance(self.filename, FSFile) + return not isinstance(self.filename, str) def _read_data_from_disk(self): # For reading the image data, unzip_context is faster than generic_open diff --git a/satpy/readers/seviri_l1b_hrit.py b/satpy/readers/seviri_l1b_hrit.py index f65faa8ecc..cd62fcf5e2 100644 --- a/satpy/readers/seviri_l1b_hrit.py +++ b/satpy/readers/seviri_l1b_hrit.py @@ -312,7 +312,7 @@ class HRITMSGPrologueEpilogueBase(HRITFileHandler): def __init__(self, filename, filename_info, filetype_info, hdr_info): """Initialize the file handler for prologue and epilogue files.""" - super(HRITMSGPrologueEpilogueBase, self).__init__(filename, filename_info, filetype_info, hdr_info) + super().__init__(filename, filename_info, filetype_info, hdr_info) self._reduced = None def _reduce(self, mda, max_size): @@ -333,11 +333,11 @@ def __init__(self, filename, filename_info, filetype_info, calib_mode="nominal", ext_calib_coefs=None, include_raw_metadata=False, mda_max_array_size=None, fill_hrv=None, mask_bad_quality_scan_lines=None): """Initialize the reader.""" - super(HRITMSGPrologueFileHandler, self).__init__(filename, filename_info, - filetype_info, - (msg_hdr_map, - msg_variable_length_headers, - msg_text_headers)) + super().__init__(filename, filename_info, + filetype_info, + (msg_hdr_map, + msg_variable_length_headers, + msg_text_headers)) self.prologue = {} self.read_prologue() diff --git a/satpy/readers/utils.py b/satpy/readers/utils.py index 983225acd5..f534960813 100644 --- a/satpy/readers/utils.py +++ b/satpy/readers/utils.py @@ -355,10 +355,10 @@ def generic_open(filename, *args, **kwargs): fp = filename.open(*args, **kwargs) except AttributeError: fp = open(filename, *args, **kwargs) - - yield fp - - fp.close() + try: + yield fp + finally: + fp.close() def fromfile(filename, dtype, count=1, offset=0): diff --git a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py index 66dc4ed0fa..55990926e3 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py @@ -502,3 +502,21 @@ def test_mask_bad_quality(self, file_handler): new_data[:, :] = np.nan expected = expected.copy(data=new_data) xr.testing.assert_equal(res, expected) + + +def test_read_real_prologue(tmp_path): + """Test reading from an actual file.""" + contents = [ + # prime header + np.void((0, 16), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((128, 90, 3403688), + dtype=[("file_type", "u1"), ("total_header_length", ">u4"), ("data_field_length", ">u8")]), + # second header + np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]) + ] + + filename = tmp_path / "prologue" + with open(filename, "wb") as fh: + for array in contents: + array.tofile(fh) + # filehandler = HRITMSGPrologueFileHandler(filename, dict(), dict()) From 0e26ea8acae7b7284e1943f612446a7c513eb870 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 13:43:53 +0100 Subject: [PATCH 02/16] Add switch for verbosity in hrit header reading --- satpy/readers/hrit_base.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index 492ba3be06..ed62b8a71f 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -124,19 +124,23 @@ def __init__(self, filename, filename_info, filetype_info, hdr_info): self._start_time = filename_info["start_time"] self._end_time = self._start_time + dt.timedelta(minutes=15) - def _get_hd(self, hdr_info): + def _get_hd(self, hdr_info, verbose=False): """Open the file, read and get the basic file header info and set the mda dictionary.""" hdr_map, variable_length_headers, text_headers = hdr_info - with utils.generic_open(self.filename, mode="rb") as fp: total_header_length = 16 while fp.tell() < total_header_length: hdr_id = get_header_id(fp) + if verbose: + print("hdr_id") # noqa: T201 + print(f'np.void({hdr_id}, dtype=[("hdr_id", "u1"), ("record_length", ">u2")]),') # noqa: T201 the_type = hdr_map[hdr_id["hdr_id"]] if the_type in variable_length_headers: field_length = int((hdr_id["record_length"] - 3) / the_type.itemsize) current_hdr = get_header_content(fp, the_type, field_length) + if verbose: + print(f"np.zeros(({field_length}, ), dtype={the_type}),") # noqa: T201 key = variable_length_headers[the_type] if key in self.mda: if not isinstance(self.mda[key], list): @@ -150,9 +154,13 @@ def _get_hd(self, hdr_info): char = list(the_type.fields.values())[0][0].char new_type = np.dtype(char + str(field_length)) current_hdr = get_header_content(fp, new_type)[0] + if verbose: + print(f'np.array({current_hdr}, dtype="{new_type}"),') # noqa: T201 self.mda[text_headers[the_type]] = current_hdr else: current_hdr = get_header_content(fp, the_type)[0] + if verbose: + print(f"np.void({current_hdr}, dtype={the_type}),") # noqa: T201 self.mda.update( dict(zip(current_hdr.dtype.names, current_hdr))) From 727d06ad87e87f3f671a15052865543e6ef010cd Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 13:45:51 +0100 Subject: [PATCH 03/16] Add real-file tests for hrit seviri --- .../reader_tests/test_seviri_l1b_hrit.py | 107 ++++++++++++++++-- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py index 55990926e3..438caec1ec 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py @@ -504,19 +504,112 @@ def test_mask_bad_quality(self, file_handler): xr.testing.assert_equal(res, expected) -def test_read_real_prologue(tmp_path): - """Test reading from an actual file.""" - contents = [ +@pytest.fixture +def prologue_file(tmp_path, prologue_header_contents): + """Create a dummy prologue file.""" + from satpy.readers.seviri_l1b_native_hdr import hrit_prologue + header = prologue_header_contents + contents = np.void(1, dtype=hrit_prologue) + contents["SatelliteStatus"]["SatelliteDefinition"]["SatelliteId"] = 324 + return create_file(tmp_path / "prologue", header + [contents]) + +@pytest.fixture +def prologue_header_contents(): + """Get the contents of the header.""" + return [ # prime header np.void((0, 16), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), np.void((128, 90, 3403688), dtype=[("file_type", "u1"), ("total_header_length", ">u4"), ("data_field_length", ">u8")]), # second header - np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]) + np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + # np.void(1, dtype=[("text", "|S61")]), + np.array(b"H-000-MSG4__-MSG4________-_________-PRO______-201802281500-__", dtype="|S61"), + # timestamp record + np.void((5, 10), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((64, (21973, 54911033)), + dtype=[("cds_p_field", "u1"), ("timestamp", [("Days", ">u2"), ("Milliseconds", ">u4")])]) ] - filename = tmp_path / "prologue" + +@pytest.fixture +def epilogue_file(tmp_path, epilogue_header_contents): + """Create a dummy epilogue file.""" + from satpy.readers.seviri_l1b_native_hdr import hrit_epilogue + header = epilogue_header_contents + contents = np.void(1, dtype=hrit_epilogue) + return create_file(tmp_path / "epilogue", header + [contents]) + + +@pytest.fixture +def epilogue_header_contents(): + """Get the contents of the header.""" + return [ + np.void((0, 16), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((129, 90, 3042600), + dtype=[("file_type", "u1"), ("total_header_length", ">u4"), ("data_field_length", ">u8")]), + np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.array(b"H-000-MSG4__-MSG4________-_________-EPI______-201802281500-__", dtype="|S61"), + np.void((5, 10), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((64, (21973, 54911033)), + dtype=[("cds_p_field", "u1"), ("timestamp", [("Days", ">u2"), ("Milliseconds", ">u4")])]), + ] + + +def create_file(filename, file_contents): + """Create an hrit file.""" with open(filename, "wb") as fh: - for array in contents: + for array in file_contents: array.tofile(fh) - # filehandler = HRITMSGPrologueFileHandler(filename, dict(), dict()) + return filename + + +@pytest.fixture +def segment_file(tmp_path): + """Create a segment_file.""" + cols = 3712 + lines = 464 + bpp = 10 + header = [ + np.void((0, 16), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((0, 6198, 17223680), dtype=[("file_type", "u1"), ("total_header_length", ">u4"), + ("data_field_length", ">u8")]), + np.void((1, 9), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((bpp, cols, lines, 0), dtype=[("number_of_bits_per_pixel", "u1"), ("number_of_columns", ">u2"), + ("number_of_lines", ">u2"), ("compression_flag_for_data", "u1")]), + np.void((2, 51), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((b"GEOS(+000.0) ", -13642337, -13642337, 1856, 1856), + dtype=[("projection_name", "S32"), + ("cfac", ">i4"), ("lfac", ">i4"), + ("coff", ">i4"), ("loff", ">i4")]), + np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.array(b"H-000-MSG4__-MSG4________-VIS008___-000001___-201802281500-__", dtype="|S61"), + np.void((5, 10), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((64, (21973, 54911033)), dtype=[("cds_p_field", "u1"), ("timestamp", [("Days", ">u2"), + ("Milliseconds", ">u4")])]), + np.void((128, 13), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.void((324, 2, 1, 1, 8, 0), dtype=[("GP_SC_ID", ">i2"), ("spectral_channel_id", "i1"), + ("segment_sequence_number", ">u2"), + ("planned_start_segment_number", ">u2"), + ("planned_end_segment_number", ">u2"), + ("data_field_representation", "i1")]), + np.void((129, 6035), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), + np.zeros((464, ), dtype=[("line_number_in_grid", ">i4"), + ("line_mean_acquisition", [("days", ">u2"), ("milliseconds", ">u4")]), + ("line_validity", "u1"), ("line_radiometric_quality", "u1"), + ("line_geometric_quality", "u1")]), + ] + contents = np.empty(cols * lines * bpp // 8, dtype="u1") + + return create_file(tmp_path / "segment", header + [contents]) + + +def test_read_real_segment(prologue_file, epilogue_file, segment_file): + """Test reading an hrit segment.""" + info = dict(start_time=dt.datetime(2018, 2, 28, 15, 0), service="") + prologue_fh = HRITMSGPrologueFileHandler(prologue_file, info, dict()) + epilogue_fh = HRITMSGEpilogueFileHandler(epilogue_file, info, dict()) + filehandler = HRITMSGFileHandler(segment_file, info, dict(), prologue_fh, epilogue_fh) + res = filehandler.get_dataset(dict(name="VIS008", calibration="counts"), + dict(units="", wavelength=0.8, standard_name="counts")) + res.compute() From 0a6ef497c0f558ec4c7f11d4e78e52e6c9e357b5 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 14:53:12 +0100 Subject: [PATCH 04/16] Improver fixture name and specify warning catching better --- satpy/tests/reader_tests/test_seviri_l1b_native.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 5c6a86596e..40d770c6d6 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1286,8 +1286,8 @@ def tmp_seviri_nat_filename(session_tmp_path): @pytest.fixture(scope="session") -def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): - """Compress the given seviri native file into a zip file.""" +def compressed_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): + """Return the fsspec path to the given seviri native file inside a zip file.""" zip_full_path = session_tmp_path / "test_seviri_native.zip" with zipfile.ZipFile(zip_full_path, mode="w") as archive: archive.write(tmp_seviri_nat_filename, os.path.basename(tmp_seviri_nat_filename)) @@ -1296,7 +1296,7 @@ def compress_seviri_native_file(tmp_seviri_nat_filename, session_tmp_path): @pytest.mark.parametrize(("full_path"), [ lf("tmp_seviri_nat_filename"), - lf("compress_seviri_native_file") + lf("compressed_seviri_native_file") ]) def test_read_physical_seviri_nat_file(full_path): """Test that the physical seviri native file can be read successfully, in case of both a plain and a zip file. @@ -1312,7 +1312,7 @@ def test_read_physical_seviri_nat_file(full_path): assert set(scene.available_dataset_names()) == set(CHANNEL_INDEX_LIST) with warnings.catch_warnings(): - warnings.filterwarnings("ignore", category=UserWarning) + warnings.filterwarnings("ignore", category=UserWarning, message="No orbit polynomial valid") scene.load(["VIS006"]) assert scene["VIS006"].dtype == np.float32 assert scene["VIS006"].values.dtype == np.float32 From 43340802e62675f0a896a9a3197e9ffa7d23f529 Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 14:55:31 +0100 Subject: [PATCH 05/16] Move session_tmp_path to conftest --- satpy/conftest.py | 7 +++++++ satpy/tests/reader_tests/test_seviri_l1b_native.py | 6 ------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/satpy/conftest.py b/satpy/conftest.py index 70843a35e9..d0ab1391a7 100644 --- a/satpy/conftest.py +++ b/satpy/conftest.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Pytest configuration and setup functions.""" +import pytest def pytest_configure(config): @@ -28,3 +29,9 @@ def pytest_unconfigure(config): """Undo previous configurations.""" from satpy import aux_download aux_download.RUNNING_TESTS = False + + +@pytest.fixture(scope="session") +def session_tmp_path(tmp_path_factory): + """Generate a single temp path to use for the entire session.""" + return tmp_path_factory.mktemp("data") diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index 40d770c6d6..3da781a112 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -1271,12 +1271,6 @@ def test_read_header(): assert actual == expected -@pytest.fixture(scope="session") -def session_tmp_path(tmp_path_factory): - """Generate a single temp path to use for the entire session.""" - return tmp_path_factory.mktemp("data") - - @pytest.fixture(scope="session") def tmp_seviri_nat_filename(session_tmp_path): """Create a fully-qualified filename for a seviri native format file.""" From a26f669edef388206a4af756f3996e354cb27a6e Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 14:56:31 +0100 Subject: [PATCH 06/16] Test remote file for hrit --- .../reader_tests/test_seviri_l1b_hrit.py | 59 +++++++++++++++---- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py index 438caec1ec..b436b23e21 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py @@ -19,9 +19,13 @@ """The HRIT msg reader tests package.""" import datetime as dt +import os import unittest +import warnings +import zipfile from unittest import mock +import fsspec import numpy as np import pytest import xarray as xr @@ -29,6 +33,7 @@ from pyproj import CRS import satpy.tests.reader_tests.test_seviri_l1b_hrit_setup as setup +from satpy.readers import FSFile from satpy.readers.seviri_l1b_hrit import HRITMSGEpilogueFileHandler, HRITMSGFileHandler, HRITMSGPrologueFileHandler from satpy.tests.reader_tests.test_seviri_base import ORBIT_POLYNOMIALS_INVALID from satpy.tests.reader_tests.test_seviri_l1b_calibration import TestFileHandlerCalibrationBase @@ -504,16 +509,17 @@ def test_mask_bad_quality(self, file_handler): xr.testing.assert_equal(res, expected) -@pytest.fixture -def prologue_file(tmp_path, prologue_header_contents): +@pytest.fixture(scope="session") +def prologue_file(session_tmp_path, prologue_header_contents): """Create a dummy prologue file.""" from satpy.readers.seviri_l1b_native_hdr import hrit_prologue header = prologue_header_contents contents = np.void(1, dtype=hrit_prologue) contents["SatelliteStatus"]["SatelliteDefinition"]["SatelliteId"] = 324 - return create_file(tmp_path / "prologue", header + [contents]) + return create_file(session_tmp_path / "prologue", header + [contents]) -@pytest.fixture + +@pytest.fixture(scope="session") def prologue_header_contents(): """Get the contents of the header.""" return [ @@ -523,7 +529,6 @@ def prologue_header_contents(): dtype=[("file_type", "u1"), ("total_header_length", ">u4"), ("data_field_length", ">u8")]), # second header np.void((4, 64), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), - # np.void(1, dtype=[("text", "|S61")]), np.array(b"H-000-MSG4__-MSG4________-_________-PRO______-201802281500-__", dtype="|S61"), # timestamp record np.void((5, 10), dtype=[("hdr_id", "u1"), ("record_length", ">u2")]), @@ -532,16 +537,16 @@ def prologue_header_contents(): ] -@pytest.fixture -def epilogue_file(tmp_path, epilogue_header_contents): +@pytest.fixture(scope="session") +def epilogue_file(session_tmp_path, epilogue_header_contents): """Create a dummy epilogue file.""" from satpy.readers.seviri_l1b_native_hdr import hrit_epilogue header = epilogue_header_contents contents = np.void(1, dtype=hrit_epilogue) - return create_file(tmp_path / "epilogue", header + [contents]) + return create_file(session_tmp_path / "epilogue", header + [contents]) -@pytest.fixture +@pytest.fixture(scope="session") def epilogue_header_contents(): """Get the contents of the header.""" return [ @@ -564,8 +569,8 @@ def create_file(filename, file_contents): return filename -@pytest.fixture -def segment_file(tmp_path): +@pytest.fixture(scope="session") +def segment_file(session_tmp_path): """Create a segment_file.""" cols = 3712 lines = 464 @@ -601,7 +606,7 @@ def segment_file(tmp_path): ] contents = np.empty(cols * lines * bpp // 8, dtype="u1") - return create_file(tmp_path / "segment", header + [contents]) + return create_file(session_tmp_path / "segment", header + [contents]) def test_read_real_segment(prologue_file, epilogue_file, segment_file): @@ -609,7 +614,35 @@ def test_read_real_segment(prologue_file, epilogue_file, segment_file): info = dict(start_time=dt.datetime(2018, 2, 28, 15, 0), service="") prologue_fh = HRITMSGPrologueFileHandler(prologue_file, info, dict()) epilogue_fh = HRITMSGEpilogueFileHandler(epilogue_file, info, dict()) - filehandler = HRITMSGFileHandler(segment_file, info, dict(), prologue_fh, epilogue_fh) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, message="No orbit polynomial valid") + filehandler = HRITMSGFileHandler(segment_file, info, dict(), prologue_fh, epilogue_fh) + res = filehandler.get_dataset(dict(name="VIS008", calibration="counts"), + dict(units="", wavelength=0.8, standard_name="counts")) + res.compute() + + +@pytest.fixture(scope="session") +def compressed_seviri_hrit_files(session_tmp_path, prologue_file, epilogue_file, segment_file): + """Return the fsspec paths to the given seviri hrit files inside a zip file.""" + zip_full_path = session_tmp_path / "test_seviri_hrit.zip" + with zipfile.ZipFile(zip_full_path, mode="w") as archive: + for filename in (prologue_file, epilogue_file, segment_file): + archive.write(filename, os.path.basename(filename)) + return {hrit_file: f"zip://{hrit_file}::file://{zip_full_path.as_posix()}" + for hrit_file in ("prologue", "epilogue", "segment")} + +def test_read_real_segment_zipped(compressed_seviri_hrit_files): + """Test reading an hrit segment.""" + info = dict(start_time=dt.datetime(2018, 2, 28, 15, 0), service="") + prologue = FSFile(fsspec.open(compressed_seviri_hrit_files["prologue"])) + prologue_fh = HRITMSGPrologueFileHandler(prologue, info, dict()) + epilogue = FSFile(fsspec.open(compressed_seviri_hrit_files["epilogue"])) + epilogue_fh = HRITMSGEpilogueFileHandler(epilogue, info, dict()) + segment = FSFile(fsspec.open(compressed_seviri_hrit_files["segment"])) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, message="No orbit polynomial valid") + filehandler = HRITMSGFileHandler(segment, info, dict(), prologue_fh, epilogue_fh) res = filehandler.get_dataset(dict(name="VIS008", calibration="counts"), dict(units="", wavelength=0.8, standard_name="counts")) res.compute() From 83e812e0710b09cdfd14e248a3855775e329ee4b Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 19:07:05 +0100 Subject: [PATCH 07/16] Fix hrit base to allow decompressing Paths and FSFiles --- satpy/readers/hrit_base.py | 25 ++++++++++++++-------- satpy/tests/reader_tests/test_hrit_base.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/satpy/readers/hrit_base.py b/satpy/readers/hrit_base.py index ed62b8a71f..910131f21b 100644 --- a/satpy/readers/hrit_base.py +++ b/satpy/readers/hrit_base.py @@ -87,14 +87,18 @@ } -def decompress(infile): +def decompress_file(infile) -> bytes: """Decompress an XRIT data file and return the decompressed buffer.""" - from pyPublicDecompWT import xRITDecompress - # decompress in-memory with open(infile, mode="rb") as fh: - xrit = xRITDecompress() - xrit.decompress(fh.read()) + return decompress_buffer(fh.read()) + + +def decompress_buffer(buffer) -> bytes: + """Decompress buffer.""" + from pyPublicDecompWT import xRITDecompress + xrit = xRITDecompress() + xrit.decompress(buffer) return xrit.data() @@ -333,7 +337,7 @@ def _read_data_from_disk(self): if self.compressed: return np.frombuffer( - decompress(fn), + decompress_file(fn), offset=self.offset, dtype=dtype, count=np.prod(shape) @@ -350,12 +354,15 @@ def _read_file_like(self): # filename is likely to be a file-like object, already in memory dtype, shape = self._get_input_info() with utils.generic_open(self.filename, mode="rb") as fp: + decompressed_buffer = fp.read() + if self.compressed: + decompressed_buffer = decompress_buffer(decompressed_buffer) no_elements = np.prod(shape) - fp.seek(self.offset) return np.frombuffer( - fp.read(np.dtype(dtype).itemsize * no_elements), + decompressed_buffer, dtype=np.dtype(dtype), - count=no_elements.item() + count=no_elements.item(), + offset=self.offset ).reshape(shape) def _get_input_info(self): diff --git a/satpy/tests/reader_tests/test_hrit_base.py b/satpy/tests/reader_tests/test_hrit_base.py index 899ebf74f4..2a18f1aa61 100644 --- a/satpy/tests/reader_tests/test_hrit_base.py +++ b/satpy/tests/reader_tests/test_hrit_base.py @@ -241,7 +241,7 @@ def test_read_band_filepath(self, stub_compressed_hrit_file): """Test reading a single band from a filepath.""" filename = stub_compressed_hrit_file - with mock.patch("satpy.readers.hrit_base.decompress", side_effect=fake_decompress) as mock_decompress: + with mock.patch("satpy.readers.hrit_base.decompress_buffer", side_effect=fake_decompress) as mock_decompress: with mock.patch.object(HRITFileHandler, "_get_hd", side_effect=new_get_hd, autospec=True) as get_hd: self.reader = HRITFileHandler(filename, {"platform_shortname": "MSG3", From 17cf0ce9f86f1bf9176719e37925f6d9f2f1106b Mon Sep 17 00:00:00 2001 From: Martin Raspaud Date: Thu, 13 Feb 2025 19:27:34 +0100 Subject: [PATCH 08/16] Test passing hrit file as UPath --- continuous_integration/environment.yaml | 1 + .../reader_tests/test_seviri_l1b_hrit.py | 33 ++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index df10c1a7a8..605a9b5bb9 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -46,6 +46,7 @@ dependencies: - pytest - pytest-cov - fsspec + - universal_pathlib - botocore>=1.33 - s3fs - python-geotiepoints diff --git a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py index b436b23e21..e67119c552 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_hrit.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_hrit.py @@ -633,13 +633,44 @@ def compressed_seviri_hrit_files(session_tmp_path, prologue_file, epilogue_file, for hrit_file in ("prologue", "epilogue", "segment")} def test_read_real_segment_zipped(compressed_seviri_hrit_files): - """Test reading an hrit segment.""" + """Test reading a remote hrit segment passed as FSFile.""" + info = dict(start_time=dt.datetime(2018, 2, 28, 15, 0), service="") + prologue = FSFile(fsspec.open(compressed_seviri_hrit_files["prologue"])) + prologue_fh = HRITMSGPrologueFileHandler(prologue, info, dict()) + epilogue = FSFile(fsspec.open(compressed_seviri_hrit_files["epilogue"])) + epilogue_fh = HRITMSGEpilogueFileHandler(epilogue, info, dict()) + segment = FSFile(fsspec.open(compressed_seviri_hrit_files["segment"])) + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning, message="No orbit polynomial valid") + filehandler = HRITMSGFileHandler(segment, info, dict(), prologue_fh, epilogue_fh) + res = filehandler.get_dataset(dict(name="VIS008", calibration="counts"), + dict(units="", wavelength=0.8, standard_name="counts")) + res.compute() + + +def to_upath(fsfile): + """Convert FSFile instance to UPath.""" + from upath import UPath + fsfile_fs = fsfile.fs.to_dict() + fsfile_fs.pop("cls") + path = UPath(os.fspath(fsfile), **fsfile_fs) + return path + + +def test_read_real_segment_zipped_with_upath(compressed_seviri_hrit_files): + """Test reading a remote hrit segment passed as UPath.""" info = dict(start_time=dt.datetime(2018, 2, 28, 15, 0), service="") + prologue = FSFile(fsspec.open(compressed_seviri_hrit_files["prologue"])) + prologue = to_upath(prologue) prologue_fh = HRITMSGPrologueFileHandler(prologue, info, dict()) + epilogue = FSFile(fsspec.open(compressed_seviri_hrit_files["epilogue"])) + epilogue = to_upath(epilogue) epilogue_fh = HRITMSGEpilogueFileHandler(epilogue, info, dict()) + segment = FSFile(fsspec.open(compressed_seviri_hrit_files["segment"])) + segment = to_upath(segment) with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=UserWarning, message="No orbit polynomial valid") filehandler = HRITMSGFileHandler(segment, info, dict(), prologue_fh, epilogue_fh) From fc11e7bfe988b7568d73836be650a3cb2c082ff0 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 19 Feb 2025 13:12:19 -0600 Subject: [PATCH 09/16] Fix deprecated codecov action parameter use --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e18d23d63e..e338e2d6f1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -118,7 +118,7 @@ jobs: uses: codecov/codecov-action@v5 with: flags: unittests - file: ./coverage.xml + files: ./coverage.xml env_vars: OS,PYTHON_VERSION,UNSTABLE - name: Coveralls Parallel From 98ce654c6640390eeeb70b58b4f47e878f5bc3d5 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 19 Feb 2025 14:04:07 -0600 Subject: [PATCH 10/16] Update to sphinx.ext.apidoc --- doc/rtd_environment.yml | 3 +-- doc/source/conf.py | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/doc/rtd_environment.yml b/doc/rtd_environment.yml index 58a1e068f5..df09fee132 100644 --- a/doc/rtd_environment.yml +++ b/doc/rtd_environment.yml @@ -25,9 +25,8 @@ dependencies: - rioxarray - setuptools - setuptools_scm - - sphinx + - sphinx>=8.2.0 - sphinx_rtd_theme - - sphinxcontrib-apidoc - trollsift - xarray - zarr diff --git a/doc/source/conf.py b/doc/source/conf.py index 4c8405d19f..9ac82d830e 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -82,11 +82,14 @@ def __getattr__(cls, name): # -- General configuration ----------------------------------------------------- +# sphinxcontrib.apidoc was added to sphinx in 8.2.0 as sphinx.etx.apidoc +needs_sphinx = "8.2.0" + # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = ["sphinx.ext.autodoc", "sphinx.ext.intersphinx", "sphinx.ext.todo", "sphinx.ext.coverage", "sphinx.ext.doctest", "sphinx.ext.napoleon", "sphinx.ext.autosummary", "sphinx.ext.autosectionlabel", - "doi_role", "sphinx.ext.viewcode", "sphinxcontrib.apidoc", + "doi_role", "sphinx.ext.viewcode", "sphinx.ext.apidoc", "sphinx.ext.mathjax"] # Autosectionlabel @@ -95,18 +98,19 @@ def __getattr__(cls, name): autosectionlabel_maxdepth = 3 # API docs -apidoc_module_dir = "../../satpy" -apidoc_output_dir = "api" -apidoc_excluded_paths = [ - "readers/caliop_l2_cloud.py", - "readers/ghrsst_l3c_sst.py", - "readers/li_l2.py", - "readers/scatsat1_l2b.py", +apidoc_modules = [ + { + "path": "../../satpy", + "destination": "api/", + "exclude_patterns": [ + "../../satpy/readers/caliop_l2_cloud.py", + "../../satpy/readers/ghrsst_l3c_sst.py", + "../../satpy/readers/scatsat1_l2b.py", + ], + }, ] apidoc_separate_modules = True -apidoc_extra_args = [ - "--private", -] +apidoc_include_private = True # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] @@ -297,7 +301,7 @@ def __getattr__(cls, name): "scipy": ("https://scipy.github.io/devdocs", None), "trollimage": ("https://trollimage.readthedocs.io/en/stable", None), "trollsift": ("https://trollsift.readthedocs.io/en/stable", None), - "xarray": ("https://xarray.pydata.org/en/stable", None), + "xarray": ("https://docs.xarray.dev/en/stable", None), "rasterio": ("https://rasterio.readthedocs.io/en/latest", None), "donfig": ("https://donfig.readthedocs.io/en/latest", None), "pooch": ("https://www.fatiando.org/pooch/latest/", None), From ac028b6ae15a8f78f78c862f5c60b653979836c2 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Feb 2025 20:22:47 -0600 Subject: [PATCH 11/16] Skip documenting tests in API docs Saves on memory usage and overall execution time for documentation that very few would use --- doc/Makefile | 1 - doc/source/conf.py | 7 +++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/doc/Makefile b/doc/Makefile index 624fe21234..eb43bdcc02 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -29,7 +29,6 @@ help: clean: -rm -rf $(BUILDDIR)/* - -rm -rf source/api/*.rst html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html diff --git a/doc/source/conf.py b/doc/source/conf.py index 9ac82d830e..637aa864d3 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -106,6 +106,13 @@ def __getattr__(cls, name): "../../satpy/readers/caliop_l2_cloud.py", "../../satpy/readers/ghrsst_l3c_sst.py", "../../satpy/readers/scatsat1_l2b.py", + # Prefer to not document test modules. Most users will look at + # source code if needed and we want to avoid documentation builds + # suffering from import-time test data creation. We want to keep + # things contributors might be interested in like satpy.tests.utils. + "../../satpy/tests/test_*.py", + "../../satpy/tests/**/test_*.py", + "../../satpy/tests/**/_*.py", ], }, ] From e2efa7fce47f562ec21a8b06f4a1799fdb26a0c7 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Feb 2025 20:52:00 -0600 Subject: [PATCH 12/16] Allow more test modules to be documented to avoid missing references --- doc/source/conf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 637aa864d3..3fae35cf67 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -112,7 +112,6 @@ def __getattr__(cls, name): # things contributors might be interested in like satpy.tests.utils. "../../satpy/tests/test_*.py", "../../satpy/tests/**/test_*.py", - "../../satpy/tests/**/_*.py", ], }, ] From 9b43c72cd942f38d2ed940c67913fa711364f913 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Feb 2025 20:57:32 -0600 Subject: [PATCH 13/16] Fix other codecov action deprecated parameter --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e338e2d6f1..cfd18ccce1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -139,7 +139,7 @@ jobs: uses: codecov/codecov-action@v5 with: flags: behaviourtests - file: ./coverage.xml + files: ./coverage.xml env_vars: OS,PYTHON_VERSION,UNSTABLE coveralls: From 07adb7ad7dcb370df02ac03ae6c2059f41c507a0 Mon Sep 17 00:00:00 2001 From: Malcolm Taberner Date: Thu, 20 Feb 2025 15:45:40 +0000 Subject: [PATCH 14/16] To get object attributes use __dict__ but retrieve all possible attributes on failure. --- satpy/readers/netcdf_utils.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/satpy/readers/netcdf_utils.py b/satpy/readers/netcdf_utils.py index c8b8a3f85f..97d9995af1 100644 --- a/satpy/readers/netcdf_utils.py +++ b/satpy/readers/netcdf_utils.py @@ -211,8 +211,21 @@ def _collect_global_attrs(self, obj): self.file_content[fc_key] = global_attrs[key] = value self.file_content["/attrs"] = global_attrs - def _get_object_attrs(self, obj): - return obj.__dict__ + @classmethod + def _get_object_attrs(cls, obj): + """Get object attributes using __dict__ but retrieve recoverable attributes on failure.""" + try: + return obj.__dict__ + except KeyError: + # Maybe unrecognised datatype. + atts = {} + for attname in obj.ncattrs(): + try: + atts[attname] = obj.getncattr(attname) + except KeyError: + LOG.warning(f"Warning: Cannot load object ({obj.name}) attribute ({attname}).") + pass + return atts def _collect_attrs(self, name, obj): """Collect all the attributes for the provided file object.""" From 66823342fb02b32cfcc3bbacabc79c4c70c3ca19 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 28 Feb 2025 08:22:13 -0600 Subject: [PATCH 15/16] Update satpy/readers/netcdf_utils.py --- satpy/readers/netcdf_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/readers/netcdf_utils.py b/satpy/readers/netcdf_utils.py index 97d9995af1..17beb1e56b 100644 --- a/satpy/readers/netcdf_utils.py +++ b/satpy/readers/netcdf_utils.py @@ -224,7 +224,6 @@ def _get_object_attrs(cls, obj): atts[attname] = obj.getncattr(attname) except KeyError: LOG.warning(f"Warning: Cannot load object ({obj.name}) attribute ({attname}).") - pass return atts def _collect_attrs(self, name, obj): From 128bb9ea3ecb1c5843c2d72ac6d070666cc5ef1c Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 28 Feb 2025 08:50:07 -0600 Subject: [PATCH 16/16] Update satpy/readers/netcdf_utils.py --- satpy/readers/netcdf_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/satpy/readers/netcdf_utils.py b/satpy/readers/netcdf_utils.py index 17beb1e56b..b659f9161b 100644 --- a/satpy/readers/netcdf_utils.py +++ b/satpy/readers/netcdf_utils.py @@ -211,8 +211,8 @@ def _collect_global_attrs(self, obj): self.file_content[fc_key] = global_attrs[key] = value self.file_content["/attrs"] = global_attrs - @classmethod - def _get_object_attrs(cls, obj): + @staticmethod + def _get_object_attrs(obj): """Get object attributes using __dict__ but retrieve recoverable attributes on failure.""" try: return obj.__dict__