diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8f6816285c..faa0aea2cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -90,7 +90,7 @@ jobs: # may break the conda-forge libraries trying to use newer glibc versions run: | python -m pip install \ - --index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple/ \ + --index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple/ \ --trusted-host pypi.anaconda.org \ --no-deps --pre --upgrade \ matplotlib \ diff --git a/.github/workflows/deploy-sdist.yaml b/.github/workflows/deploy-sdist.yaml index a32d6a53dc..68463b2b03 100644 --- a/.github/workflows/deploy-sdist.yaml +++ b/.github/workflows/deploy-sdist.yaml @@ -19,7 +19,7 @@ jobs: - name: Publish package to PyPI if: github.event.action == 'published' - uses: pypa/gh-action-pypi-publish@v1.8.6 + uses: pypa/gh-action-pypi-publish@v1.8.8 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b6be918dd..799ae0a867 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,67 @@ +## Version 0.43.0 (2023/07/03) + +### Issues Closed + +* [Issue 2519](https://github.com/pytroll/satpy/issues/2519) - MSG Dust RGB adding coastilnes and grid to the image +* [Issue 2506](https://github.com/pytroll/satpy/issues/2506) - Add xarray_kwargs capability to the geocat reader ([PR 2507](https://github.com/pytroll/satpy/pull/2507) by [@joleenf](https://github.com/joleenf)) +* [Issue 2502](https://github.com/pytroll/satpy/issues/2502) - Cropping S3 image not working +* [Issue 2494](https://github.com/pytroll/satpy/issues/2494) - avhrr_l1b_gaclac fails to read most files from NOAA CLASS ([PR 2501](https://github.com/pytroll/satpy/pull/2501) by [@sfinkens](https://github.com/sfinkens)) +* [Issue 2490](https://github.com/pytroll/satpy/issues/2490) - ninjogeotiff writer adds offset/scale factor when this is not meaningful ([PR 2491](https://github.com/pytroll/satpy/pull/2491) by [@gerritholl](https://github.com/gerritholl)) +* [Issue 2483](https://github.com/pytroll/satpy/issues/2483) - Cacheing doesn't work with `scn.crop` ([PR 2485](https://github.com/pytroll/satpy/pull/2485) by [@djhoese](https://github.com/djhoese)) +* [Issue 2465](https://github.com/pytroll/satpy/issues/2465) - Possibility of dual licensing: GPL-3.0 & MIT +* [Issue 2464](https://github.com/pytroll/satpy/issues/2464) - MITIFF writer using pillow: turn off compression due to rowsperstrip issues +* [Issue 2463](https://github.com/pytroll/satpy/issues/2463) - seviri_l1b_native reader issue with reading remote files (azure) +* [Issue 2409](https://github.com/pytroll/satpy/issues/2409) - Inconsistent behavior of time attributes in EUM L1 GEO readers ([PR 2420](https://github.com/pytroll/satpy/pull/2420) by [@YouvaEUMex](https://github.com/YouvaEUMex)) +* [Issue 1749](https://github.com/pytroll/satpy/issues/1749) - Load from blended scene +* [Issue 859](https://github.com/pytroll/satpy/issues/859) - Doesn't recognize MODIS L2 file + +In this release 12 issues were closed. + +### Pull Requests Merged + +#### Bugs fixed + +* [PR 2522](https://github.com/pytroll/satpy/pull/2522) - Fix CF tests due to new xarray release +* [PR 2516](https://github.com/pytroll/satpy/pull/2516) - Fix SEVIRI native reader failing when missing main header +* [PR 2510](https://github.com/pytroll/satpy/pull/2510) - Fix warnings from NWCSAF reader +* [PR 2507](https://github.com/pytroll/satpy/pull/2507) - Fix HDF4 support in geocat reader with hardcoded engine ([2506](https://github.com/pytroll/satpy/issues/2506)) +* [PR 2492](https://github.com/pytroll/satpy/pull/2492) - Fix xarray version for cf tests +* [PR 2491](https://github.com/pytroll/satpy/pull/2491) - Change logic for ninjogeotiff gradient/axisintercept tags ([2490](https://github.com/pytroll/satpy/issues/2490)) +* [PR 2485](https://github.com/pytroll/satpy/pull/2485) - Fix angle caching not handling a specific type of irregular chunking ([2483](https://github.com/pytroll/satpy/issues/2483)) +* [PR 2481](https://github.com/pytroll/satpy/pull/2481) - Fix NWCSAF reading for NOAA-21 + +#### Features added + +* [PR 2521](https://github.com/pytroll/satpy/pull/2521) - Add a median filter modifier +* [PR 2508](https://github.com/pytroll/satpy/pull/2508) - Add support for OLCI L2 files which are missing Frame_IDs +* [PR 2504](https://github.com/pytroll/satpy/pull/2504) - Improve flexibility of olci level2 reader +* [PR 2501](https://github.com/pytroll/satpy/pull/2501) - Add Pygac reference to avhrr_l1b_gaclac documentation ([2494](https://github.com/pytroll/satpy/issues/2494)) +* [PR 2499](https://github.com/pytroll/satpy/pull/2499) - Add option to clip negative ABI radiances +* [PR 2497](https://github.com/pytroll/satpy/pull/2497) - Enable to pass a custom function to Scene.aggregate +* [PR 2489](https://github.com/pytroll/satpy/pull/2489) - Add "neutral_resolution_band" kwarg to RatioSharpenedRGB/SelfSharpenedRGB +* [PR 2480](https://github.com/pytroll/satpy/pull/2480) - Add helper-function for reading SEVIRI L1.5 Native header. +* [PR 2449](https://github.com/pytroll/satpy/pull/2449) - Generalise the `true_color_reproduction` composite and enhancement +* [PR 2420](https://github.com/pytroll/satpy/pull/2420) - Fix inconsistent behavior of time attributes in EUM L1 GEO readers ([2409](https://github.com/pytroll/satpy/issues/2409)) +* [PR 2259](https://github.com/pytroll/satpy/pull/2259) - Refactor `CFWriter.save_datasets` and enable retrieval of equivalent xr.Dataset with `scn.to_xarray()` +* [PR 2117](https://github.com/pytroll/satpy/pull/2117) - Add reader for GMS-5 VISSR data + +#### Documentation changes + +* [PR 2514](https://github.com/pytroll/satpy/pull/2514) - Fix argument name in DayNightComposite example document +* [PR 2501](https://github.com/pytroll/satpy/pull/2501) - Add Pygac reference to avhrr_l1b_gaclac documentation ([2494](https://github.com/pytroll/satpy/issues/2494)) +* [PR 2478](https://github.com/pytroll/satpy/pull/2478) - Fix eccodes package names in setup.py, update documentation for setting up development environment. +* [PR 2474](https://github.com/pytroll/satpy/pull/2474) - Reorganize seviri_l2_grib.yaml file and add more documentation to seviri_l1b_native.py + +#### Clean ups + +* [PR 2523](https://github.com/pytroll/satpy/pull/2523) - Convert CF Writer tests to pytest +* [PR 2486](https://github.com/pytroll/satpy/pull/2486) - Fix leftover deprecated nosetest teardown methods +* [PR 2478](https://github.com/pytroll/satpy/pull/2478) - Fix eccodes package names in setup.py, update documentation for setting up development environment. +* [PR 2474](https://github.com/pytroll/satpy/pull/2474) - Reorganize seviri_l2_grib.yaml file and add more documentation to seviri_l1b_native.py + +In this release 28 pull requests were closed. + + ## Version 0.42.2 (2023/05/10) ### Issues Closed diff --git a/doc/source/dev_guide/index.rst b/doc/source/dev_guide/index.rst index 72068aab62..e877fd1c63 100644 --- a/doc/source/dev_guide/index.rst +++ b/doc/source/dev_guide/index.rst @@ -49,7 +49,7 @@ can do this using conda_:: .. _conda: https://conda.io/ -This will create a new environment called "satpy-dev" with Python 3.8 +This will create a new environment called "satpy-dev" with Python 3.11 installed. The second command will activate the environment so any future conda, python, or pip commands will use this new environment. @@ -125,7 +125,7 @@ Satpy's documentation is built using Sphinx. All documentation lives in the ``doc/`` directory of the project repository. For building the documentation, additional packages are needed. These can be installed with :: - pip install -e ".[all]". + pip install -e ".[all]" After editing the source files there the documentation can be generated locally:: diff --git a/satpy/tests/reader_tests/gms/test_gms5_vissr_l1b.py b/satpy/tests/reader_tests/gms/test_gms5_vissr_l1b.py index 31482f1e10..f4908c0a2b 100644 --- a/satpy/tests/reader_tests/gms/test_gms5_vissr_l1b.py +++ b/satpy/tests/reader_tests/gms/test_gms5_vissr_l1b.py @@ -9,14 +9,20 @@ import xarray as xr from pyresample.geometry import AreaDefinition -import satpy.readers.gms.gms5_vissr_format as fmt -import satpy.readers.gms.gms5_vissr_l1b as vissr -import satpy.readers.gms.gms5_vissr_navigation as nav import satpy.tests.reader_tests.gms.test_gms5_vissr_data as real_world from satpy.readers import FSFile -from satpy.tests.reader_tests.utils import get_jit_methods +from satpy.tests.reader_tests.utils import get_jit_methods, skip_numba_unstable_if_missing from satpy.tests.utils import make_dataid +try: + import satpy.readers.gms.gms5_vissr_format as fmt + import satpy.readers.gms.gms5_vissr_l1b as vissr + import satpy.readers.gms.gms5_vissr_navigation as nav +except ImportError as err: + if skip_numba_unstable_if_missing(): + pytest.skip(f"Numba is not compatible with unstable NumPy: {err!s}", allow_module_level=True) + raise + @pytest.fixture(params=[False, True], autouse=True) def disable_jit(request, monkeypatch): diff --git a/satpy/tests/reader_tests/gms/test_gms5_vissr_navigation.py b/satpy/tests/reader_tests/gms/test_gms5_vissr_navigation.py index 47c5fd044c..144139a07a 100644 --- a/satpy/tests/reader_tests/gms/test_gms5_vissr_navigation.py +++ b/satpy/tests/reader_tests/gms/test_gms5_vissr_navigation.py @@ -3,8 +3,14 @@ import numpy as np import pytest -import satpy.readers.gms.gms5_vissr_navigation as nav -from satpy.tests.reader_tests.utils import get_jit_methods +from satpy.tests.reader_tests.utils import get_jit_methods, skip_numba_unstable_if_missing + +try: + import satpy.readers.gms.gms5_vissr_navigation as nav +except ImportError as err: + if skip_numba_unstable_if_missing(): + pytest.skip(f"Numba is not compatible with unstable NumPy: {err!s}", allow_module_level=True) + raise # Navigation references computed with JMA's Msial library (files # VISSR_19960217_2331_IR1.A.IMG and VISSR_19960217_2331_VIS.A.IMG). The VIS diff --git a/satpy/tests/reader_tests/test_seviri_l1b_native.py b/satpy/tests/reader_tests/test_seviri_l1b_native.py index aaf8dc07e2..e344d09ff9 100644 --- a/satpy/tests/reader_tests/test_seviri_l1b_native.py +++ b/satpy/tests/reader_tests/test_seviri_l1b_native.py @@ -502,484 +502,276 @@ } -class TestNativeMSGFileHandler(unittest.TestCase): - """Test the NativeMSGFileHandler.""" +def create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan, good_qual='OK'): + """Create test header for SEVIRI L1.5 product. - def test_get_available_channels(self): - """Test the derivation of the available channel list.""" - available_chs = get_available_channels(TEST1_HEADER_CHNLIST) - trues = ['WV_062', 'WV_073', 'IR_108', 'VIS006', 'VIS008', 'IR_120'] - for bandname in AVAILABLE_CHANNELS: - if bandname in trues: - self.assertTrue(available_chs[bandname]) - else: - self.assertFalse(available_chs[bandname]) + Header includes mandatory attributes for NativeMSGFileHandler.get_area_extent + """ + if dataset_id['name'] == 'HRV': + reference_grid = 'ReferenceGridHRV' + column_dir_grid_step = 1.0001343488693237 + line_dir_grid_step = 1.0001343488693237 + else: + reference_grid = 'ReferenceGridVIS_IR' + column_dir_grid_step = 3.0004031658172607 + line_dir_grid_step = 3.0004031658172607 + + if is_full_disk: + north = 3712 + east = 1 + west = 3712 + south = 1 + n_visir_cols = 3712 + n_visir_lines = 3712 + n_hrv_cols = 11136 + n_hrv_lines = 11136 + ssp_lon = 0 + elif is_rapid_scan: + north = 3712 + east = 1 + west = 3712 + south = 2321 + n_visir_cols = 3712 + n_visir_lines = 1392 + n_hrv_cols = 11136 + n_hrv_lines = 4176 + ssp_lon = 9.5 + else: + north = 3574 + east = 78 + west = 2591 + south = 1746 + n_visir_cols = 2516 + n_visir_lines = north - south + 1 + n_hrv_cols = n_visir_cols * 3 + n_hrv_lines = n_visir_lines * 3 + ssp_lon = 0 + header = { + '15_MAIN_PRODUCT_HEADER': { + 'QQOV': {'Name': 'QQOV', + 'Value': good_qual} + }, + '15_DATA_HEADER': { + 'ImageDescription': { + reference_grid: { + 'ColumnDirGridStep': column_dir_grid_step, + 'LineDirGridStep': line_dir_grid_step, + 'GridOrigin': 2, # south-east corner + }, + 'ProjectionDescription': { + 'LongitudeOfSSP': ssp_lon + } + }, + 'GeometricProcessing': { + 'EarthModel': { + 'TypeOfEarthModel': earth_model, + 'EquatorialRadius': 6378169.0, + 'NorthPolarRadius': 6356583.800000001, + 'SouthPolarRadius': 6356583.800000001, + } + }, + 'SatelliteStatus': { + 'SatelliteDefinition': { + 'SatelliteId': 324 + } + } + }, + '15_SECONDARY_PRODUCT_HEADER': { + 'NorthLineSelectedRectangle': {'Value': north}, + 'EastColumnSelectedRectangle': {'Value': east}, + 'WestColumnSelectedRectangle': {'Value': west}, + 'SouthLineSelectedRectangle': {'Value': south}, + 'SelectedBandIDs': {'Value': 'xxxxxxxxxxxx'}, + 'NumberColumnsVISIR': {'Value': n_visir_cols}, + 'NumberLinesVISIR': {'Value': n_visir_lines}, + 'NumberColumnsHRV': {'Value': n_hrv_cols}, + 'NumberLinesHRV': {'Value': n_hrv_lines}, + } - available_chs = get_available_channels(TEST2_HEADER_CHNLIST) - trues = ['VIS006', 'VIS008', 'IR_039', 'WV_062', 'WV_073', 'IR_087', 'HRV'] - for bandname in AVAILABLE_CHANNELS: - if bandname in trues: - self.assertTrue(available_chs[bandname]) - else: - self.assertFalse(available_chs[bandname]) + } - available_chs = get_available_channels(TEST3_HEADER_CHNLIST) - for bandname in AVAILABLE_CHANNELS: - self.assertTrue(available_chs[bandname]) + return header -class TestNativeMSGArea(unittest.TestCase): - """Test NativeMSGFileHandler.get_area_extent. +def create_test_trailer(is_rapid_scan): + """Create test trailer for SEVIRI L1.5 product. - The expected results have been verified by manually - inspecting the output of geoferenced imagery. + Trailer includes mandatory attributes for NativeMSGFileHandler.get_area_extent """ - - @staticmethod - def create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan, good_qual='OK'): - """Create mocked NativeMSGFileHandler. - - Contains sufficient attributes for NativeMSGFileHandler.get_area_extent to be able to execute. - """ - if dataset_id['name'] == 'HRV': - reference_grid = 'ReferenceGridHRV' - column_dir_grid_step = 1.0001343488693237 - line_dir_grid_step = 1.0001343488693237 - else: - reference_grid = 'ReferenceGridVIS_IR' - column_dir_grid_step = 3.0004031658172607 - line_dir_grid_step = 3.0004031658172607 - - if is_full_disk: - north = 3712 - east = 1 - west = 3712 - south = 1 - n_visir_cols = 3712 - n_visir_lines = 3712 - n_hrv_cols = 11136 - n_hrv_lines = 11136 - ssp_lon = 0 - elif is_rapid_scan: - north = 3712 - east = 1 - west = 3712 - south = 2321 - n_visir_cols = 3712 - n_visir_lines = 1392 - n_hrv_cols = 11136 - n_hrv_lines = 4176 - ssp_lon = 9.5 - else: - north = 3574 - east = 78 - west = 2591 - south = 1746 - n_visir_cols = 2516 - n_visir_lines = north - south + 1 - n_hrv_cols = n_visir_cols * 3 - n_hrv_lines = n_visir_lines * 3 - ssp_lon = 0 - header = { - '15_MAIN_PRODUCT_HEADER': { - 'QQOV': {'Name': 'QQOV', - 'Value': good_qual} - }, - '15_DATA_HEADER': { - 'ImageDescription': { - reference_grid: { - 'ColumnDirGridStep': column_dir_grid_step, - 'LineDirGridStep': line_dir_grid_step, - 'GridOrigin': 2, # south-east corner - }, - 'ProjectionDescription': { - 'LongitudeOfSSP': ssp_lon - } + trailer = { + '15TRAILER': { + 'ImageProductionStats': { + 'ActualL15CoverageHRV': { + 'UpperNorthLineActual': 11136, + 'UpperWestColumnActual': 7533, + 'UpperSouthLineActual': 8193, + 'UpperEastColumnActual': 1966, + 'LowerNorthLineActual': 8192, + 'LowerWestColumnActual': 5568, + 'LowerSouthLineActual': 1, + 'LowerEastColumnActual': 1 }, - 'GeometricProcessing': { - 'EarthModel': { - 'TypeOfEarthModel': earth_model, - 'EquatorialRadius': 6378169.0, - 'NorthPolarRadius': 6356583.800000001, - 'SouthPolarRadius': 6356583.800000001, - } - }, - 'SatelliteStatus': { - 'SatelliteDefinition': { - 'SatelliteId': 324 - } + 'ActualScanningSummary': { + 'ReducedScan': is_rapid_scan } - }, - '15_SECONDARY_PRODUCT_HEADER': { - 'NorthLineSelectedRectangle': {'Value': north}, - 'EastColumnSelectedRectangle': {'Value': east}, - 'WestColumnSelectedRectangle': {'Value': west}, - 'SouthLineSelectedRectangle': {'Value': south}, - 'SelectedBandIDs': {'Value': 'xxxxxxxxxxxx'}, - 'NumberColumnsVISIR': {'Value': n_visir_cols}, - 'NumberLinesVISIR': {'Value': n_visir_lines}, - 'NumberColumnsHRV': {'Value': n_hrv_cols}, - 'NumberLinesHRV': {'Value': n_hrv_lines}, } - } + } - return header + return trailer - @staticmethod - def create_test_trailer(is_rapid_scan): - """Create Test Trailer. - Mocked Trailer with sufficient attributes for - NativeMSGFileHandler.get_area_extent to be able to execute. - """ - trailer = { - '15TRAILER': { - 'ImageProductionStats': { - 'ActualL15CoverageHRV': { - 'UpperNorthLineActual': 11136, - 'UpperWestColumnActual': 7533, - 'UpperSouthLineActual': 8193, - 'UpperEastColumnActual': 1966, - 'LowerNorthLineActual': 8192, - 'LowerWestColumnActual': 5568, - 'LowerSouthLineActual': 1, - 'LowerEastColumnActual': 1 - }, - 'ActualScanningSummary': { - 'ReducedScan': is_rapid_scan - } - } - } - } +def prepare_area_definitions(test_dict): + """Prepare calculated and expected area definitions for equal checking.""" + earth_model = test_dict['earth_model'] + dataset_id = test_dict['dataset_id'] + is_full_disk = test_dict['is_full_disk'] + is_rapid_scan = test_dict['is_rapid_scan'] + fill_disk = test_dict['fill_disk'] + header = create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan) + trailer = create_test_trailer(is_rapid_scan) + expected_area_def = test_dict['expected_area_def'] - return trailer + with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \ + mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \ + mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \ + mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \ + mock.patch( + 'satpy.readers.seviri_l1b_native.has_archive_header' + ) as has_archive_header: + has_archive_header.return_value = True + fromfile.return_value = header + recarray2dict.side_effect = (lambda x: x) + _get_memmap.return_value = np.arange(3) + fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) + fh.fill_disk = fill_disk + fh.header = header + fh.trailer = trailer + fh.image_boundaries = ImageBoundaries(header, trailer, fh.mda) + actual_area_def = fh.get_area_def(dataset_id) - def prepare_area_defs(self, test_dict): - """Prepare calculated and expected area definitions for equal checking.""" - earth_model = test_dict['earth_model'] - dataset_id = test_dict['dataset_id'] - is_full_disk = test_dict['is_full_disk'] - is_rapid_scan = test_dict['is_rapid_scan'] - fill_disk = test_dict['fill_disk'] - header = self.create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan) - trailer = self.create_test_trailer(is_rapid_scan) - expected_area_def = test_dict['expected_area_def'] - - with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \ - mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \ - mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \ - mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \ - mock.patch( - 'satpy.readers.seviri_l1b_native.has_archive_header' - ) as has_archive_header: - has_archive_header.return_value = True - fromfile.return_value = header - recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) - fh = NativeMSGFileHandler(None, {}, None) - fh.fill_disk = fill_disk - fh.header = header - fh.trailer = trailer - fh.image_boundaries = ImageBoundaries(header, trailer, fh.mda) - calc_area_def = fh.get_area_def(dataset_id) + return actual_area_def, expected_area_def - return calc_area_def, expected_area_def - # Earth model 1 tests - def test_earthmodel1_visir_fulldisk(self): - """Test the VISIR FES with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_VISIR_FULLDISK - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_hrv_fulldisk(self): - """Test the HRV FES with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_FULLDISK - ) - np.testing.assert_allclose(np.array(calculated.defs[0].area_extent), - np.array(expected['Area extent 0'])) - np.testing.assert_allclose(np.array(calculated.defs[1].area_extent), - np.array(expected['Area extent 1'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.defs[0].area_id, expected['Area ID']) - self.assertEqual(calculated.defs[1].area_id, expected['Area ID']) - - def test_earthmodel1_hrv_fulldisk_fill(self): - """Test the HRV FES padded to fulldisk with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_FULLDISK_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_visir_rapidscan(self): - """Test the VISIR RSS with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_VISIR_RAPIDSCAN - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_visir_rapidscan_fill(self): - """Test the VISIR RSS padded to fulldisk with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_VISIR_RAPIDSCAN_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_hrv_rapidscan(self): - """Test the HRV RSS with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_RAPIDSCAN - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_hrv_rapidscan_fill(self): - """Test the HRV RSS padded to fulldisk with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_RAPIDSCAN_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_visir_roi(self): - """Test the VISIR ROI with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_VISIR_ROI - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_visir_roi_fill(self): - """Test the VISIR ROI padded to fulldisk with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_VISIR_ROI_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_hrv_roi(self): - """Test the HRV ROI with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_ROI - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel1_hrv_roi_fill(self): - """Test the HRV ROI padded to fulldisk with the EarthModel 1.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL1_HRV_ROI_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - # Earth model 2 tests - def test_earthmodel2_visir_fulldisk(self): - """Test the VISIR FES with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_VISIR_FULLDISK - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_hrv_fulldisk(self): - """Test the HRV FES with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_FULLDISK - ) - np.testing.assert_allclose(np.array(calculated.defs[0].area_extent), - np.array(expected['Area extent 0'])) - np.testing.assert_allclose(np.array(calculated.defs[1].area_extent), - np.array(expected['Area extent 1'])) - - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.defs[0].area_id, expected['Area ID']) - self.assertEqual(calculated.defs[1].area_id, expected['Area ID']) - - def test_earthmodel2_hrv_fulldisk_fill(self): - """Test the HRV FES padded to fulldisk with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_FULLDISK_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_visir_rapidscan(self): - """Test the VISIR RSS with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_VISIR_RAPIDSCAN - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_visir_rapidscan_fill(self): - """Test the VISIR RSS padded to fulldisk with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_VISIR_RAPIDSCAN_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_hrv_rapidscan(self): - """Test the HRV RSS with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_RAPIDSCAN - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_hrv_rapidscan_fill(self): - """Test the HRV RSS padded to fulldisk with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_RAPIDSCAN_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_visir_roi(self): - """Test the VISIR ROI with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_VISIR_ROI - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_visir_roi_fill(self): - """Test the VISIR ROI padded to fulldisk with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_VISIR_ROI_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_hrv_roi(self): - """Test the HRV ROI with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_ROI - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - def test_earthmodel2_hrv_roi_fill(self): - """Test the HRV ROI padded to fulldisk with the EarthModel 2.""" - calculated, expected = self.prepare_area_defs( - TEST_AREA_EXTENT_EARTHMODEL2_HRV_ROI_FILL - ) - np.testing.assert_allclose(np.array(calculated.area_extent), - np.array(expected['Area extent'])) - self.assertEqual(calculated.width, expected['Number of columns']) - self.assertEqual(calculated.height, expected['Number of rows']) - self.assertEqual(calculated.area_id, expected['Area ID']) - - # Test check for Region Of Interest (ROI) data - def prepare_is_roi(self, test_dict): - """Prepare calculated and expected check for region of interest data for equal checking.""" - earth_model = 2 - dataset_id = make_dataid(name='VIS006') - is_full_disk = test_dict['is_full_disk'] - is_rapid_scan = test_dict['is_rapid_scan'] - header = self.create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan) - trailer = self.create_test_trailer(is_rapid_scan) - expected_is_roi = test_dict['is_roi'] - - with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \ - mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \ - mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \ - mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \ - mock.patch( - 'satpy.readers.seviri_l1b_native.has_archive_header' - ) as has_archive_header: - has_archive_header.return_value = True - fromfile.return_value = header - recarray2dict.side_effect = (lambda x: x) - _get_memmap.return_value = np.arange(3) - fh = NativeMSGFileHandler(None, {}, None) - fh.header = header - fh.trailer = trailer - calc_is_roi = fh.is_roi() +@pytest.mark.parametrize( + "actual, expected", + ( + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_VISIR_FULLDISK)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_FULLDISK_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_VISIR_RAPIDSCAN)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_VISIR_RAPIDSCAN_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_RAPIDSCAN)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_RAPIDSCAN_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_VISIR_ROI)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_VISIR_ROI_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_ROI)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_ROI_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_VISIR_FULLDISK)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_FULLDISK_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_VISIR_RAPIDSCAN)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_VISIR_RAPIDSCAN_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_RAPIDSCAN)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_RAPIDSCAN_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_VISIR_ROI)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_VISIR_ROI_FILL)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_ROI)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_ROI_FILL)), + ) +) +def test_area_definitions(actual, expected): + """Test area definitions with only one area.""" + np.testing.assert_allclose(np.array(actual.area_extent), + np.array(expected['Area extent'])) + assert actual.width == expected['Number of columns'] + assert actual.height == expected['Number of rows'] + assert actual.area_id == expected['Area ID'] + + +@pytest.mark.parametrize( + "actual, expected", + ( + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL1_HRV_FULLDISK)), + (prepare_area_definitions(TEST_AREA_EXTENT_EARTHMODEL2_HRV_FULLDISK)), + ) +) +def test_stacked_area_definitions(actual, expected): + """Test area definitions with stacked areas.""" + np.testing.assert_allclose(np.array(actual.defs[0].area_extent), + np.array(expected['Area extent 0'])) + np.testing.assert_allclose(np.array(actual.defs[1].area_extent), + np.array(expected['Area extent 1'])) + assert actual.width == expected['Number of columns'] + assert actual.height == expected['Number of rows'] + assert actual.defs[0].area_id, expected['Area ID'] + assert actual.defs[1].area_id, expected['Area ID'] + + +def prepare_is_roi(test_dict): + """Prepare calculated and expected check for region of interest data for equal checking.""" + earth_model = 2 + dataset_id = make_dataid(name='VIS006') + is_full_disk = test_dict['is_full_disk'] + is_rapid_scan = test_dict['is_rapid_scan'] + header = create_test_header(earth_model, dataset_id, is_full_disk, is_rapid_scan) + trailer = create_test_trailer(is_rapid_scan) + expected = test_dict['is_roi'] + + with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile, \ + mock.patch('satpy.readers.seviri_l1b_native.recarray2dict') as recarray2dict, \ + mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._get_memmap') as _get_memmap, \ + mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler._read_trailer'), \ + mock.patch( + 'satpy.readers.seviri_l1b_native.has_archive_header' + ) as has_archive_header: + has_archive_header.return_value = True + fromfile.return_value = header + recarray2dict.side_effect = (lambda x: x) + _get_memmap.return_value = np.arange(3) + fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) + fh.header = header + fh.trailer = trailer + actual = fh.is_roi() + + return actual, expected + - return calc_is_roi, expected_is_roi +@pytest.mark.parametrize( + "actual, expected", + ( + (prepare_is_roi(TEST_IS_ROI_FULLDISK)), + (prepare_is_roi(TEST_IS_ROI_RAPIDSCAN)), + (prepare_is_roi(TEST_IS_ROI_ROI)), + ) +) +def test_is_roi(actual, expected): + """Test if given area is of area-of-interest.""" + assert actual == expected + + +class TestNativeMSGFileHandler(unittest.TestCase): + """Test the NativeMSGFileHandler.""" - def test_is_roi_fulldisk(self): - """Test check for region of interest with FES data.""" - calculated, expected = self.prepare_is_roi(TEST_IS_ROI_FULLDISK) - self.assertEqual(calculated, expected) + def test_get_available_channels(self): + """Test the derivation of the available channel list.""" + available_chs = get_available_channels(TEST1_HEADER_CHNLIST) + trues = ('WV_062', 'WV_073', 'IR_108', 'VIS006', 'VIS008', 'IR_120') + for bandname in AVAILABLE_CHANNELS: + if bandname in trues: + self.assertTrue(available_chs[bandname]) + else: + self.assertFalse(available_chs[bandname]) - def test_is_roi_rapidscan(self): - """Test check for region of interest with RSS data.""" - calculated, expected = self.prepare_is_roi(TEST_IS_ROI_RAPIDSCAN) - self.assertEqual(calculated, expected) + available_chs = get_available_channels(TEST2_HEADER_CHNLIST) + trues = ('VIS006', 'VIS008', 'IR_039', 'WV_062', 'WV_073', 'IR_087', 'HRV') + for bandname in AVAILABLE_CHANNELS: + if bandname in trues: + self.assertTrue(available_chs[bandname]) + else: + self.assertFalse(available_chs[bandname]) - def test_is_roi_roi(self): - """Test check for region of interest with ROI data.""" - calculated, expected = self.prepare_is_roi(TEST_IS_ROI_ROI) - self.assertEqual(calculated, expected) + available_chs = get_available_channels(TEST3_HEADER_CHNLIST) + for bandname in AVAILABLE_CHANNELS: + self.assertTrue(available_chs[bandname]) TEST_HEADER_CALIB = { @@ -1029,7 +821,7 @@ def file_handler(self): header['15_DATA_HEADER'].update(TEST_HEADER_CALIB) with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler.__init__', return_value=None): - fh = NativeMSGFileHandler(filename='', filename_info=dict(), filetype_info=None) + fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.header = header fh.trailer = trailer fh.platform_id = self.platform_id @@ -1037,7 +829,7 @@ def file_handler(self): @pytest.mark.parametrize( ('channel', 'calibration', 'calib_mode', 'use_ext_coefs'), - [ + ( # VIS channel, internal coefficients ('VIS006', 'counts', 'NOMINAL', False), ('VIS006', 'radiance', 'NOMINAL', False), @@ -1063,7 +855,7 @@ def file_handler(self): # HRV channel, external coefficients (mode should have no effect) ('HRV', 'radiance', 'GSICS', True), ('HRV', 'reflectance', 'NOMINAL', True), - ] + ) ) def test_calibrate( self, file_handler, counts, channel, calibration, calib_mode, @@ -1122,7 +914,7 @@ def file_handler(self): data = self._fake_data() with mock.patch('satpy.readers.seviri_l1b_native.NativeMSGFileHandler.__init__', return_value=None): - fh = NativeMSGFileHandler(filename='', filename_info=dict(), filetype_info=None) + fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fh.header = header fh.trailer = trailer fh.mda = mda @@ -1197,13 +989,13 @@ def test_get_dataset(self, file_handler): 'wavelength': (1, 2, 3), 'standard_name': 'counts' } - dataset = file_handler.get_dataset(dataset_id, dataset_info) + xarr = file_handler.get_dataset(dataset_id, dataset_info) expected = self._exp_data_array() - xr.testing.assert_equal(dataset, expected) - assert 'raw_metadata' not in dataset.attrs + xr.testing.assert_equal(xarr, expected) + assert 'raw_metadata' not in xarr.attrs assert file_handler.start_time == datetime(2006, 1, 1, 12, 15, 0) assert file_handler.end_time == datetime(2006, 1, 1, 12, 30, 0) - assert_attrs_equal(dataset.attrs, expected.attrs, tolerance=1e-4) + assert_attrs_equal(xarr.attrs, expected.attrs, tolerance=1e-4) def test_time(self, file_handler): """Test start/end nominal/observation time handling.""" @@ -1230,7 +1022,7 @@ def _exp_data_array(): [44., 192., 835., 527.], [64., 273., 132., 788.]], dtype=np.float32), - dims=('y', 'x'), + dims=['y', 'x'], attrs={ 'orbital_parameters': { 'satellite_actual_longitude': -3.55117540817073, @@ -1275,8 +1067,8 @@ def test_get_dataset_with_raw_metadata(self, file_handler): 'wavelength': (1, 2, 3), 'standard_name': 'counts' } - res = file_handler.get_dataset(dataset_id, dataset_info) - assert 'raw_metadata' in res.attrs + xarr = file_handler.get_dataset(dataset_id, dataset_info) + assert 'raw_metadata' in xarr.attrs def test_satpos_no_valid_orbit_polynomial(self, file_handler): """Test satellite position if there is no valid orbit polynomial.""" @@ -1293,8 +1085,8 @@ def test_satpos_no_valid_orbit_polynomial(self, file_handler): 'standard_name': 'counts' } with pytest.warns(UserWarning, match="No orbit polynomial"): - res = file_handler.get_dataset(dataset_id, dataset_info) - assert 'satellite_actual_longitude' not in res.attrs[ + xarr = file_handler.get_dataset(dataset_id, dataset_info) + assert 'satellite_actual_longitude' not in xarr.attrs[ 'orbital_parameters'] @@ -1359,14 +1151,14 @@ def test_file_pattern(self, reader): @pytest.mark.parametrize( 'file_content,exp_header_size', - [ + ( (ASCII_STARTSWITH, 450400), # with ascii header (b'foobar', 445286), # without ascii header - ] + ) ) def test_header_type(file_content, exp_header_size): """Test identification of the file header type.""" - header = TestNativeMSGArea.create_test_header( + header = create_test_header( dataset_id=make_dataid(name='VIS006', resolution=3000), earth_model=1, is_full_disk=True, @@ -1382,21 +1174,21 @@ def test_header_type(file_content, exp_header_size): fromfile.return_value = header recarray2dict.side_effect = (lambda x: x) _get_memmap.return_value = np.arange(3) - fh = NativeMSGFileHandler('myfile', {}, None) + fh = NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) assert fh.header_type.itemsize == exp_header_size assert '15_SECONDARY_PRODUCT_HEADER' in fh.header def test_header_warning(): """Test warning is raised for NOK quality flag.""" - header_good = TestNativeMSGArea.create_test_header( + header_good = create_test_header( dataset_id=make_dataid(name='VIS006', resolution=3000), earth_model=1, is_full_disk=True, is_rapid_scan=0, good_qual='OK' ) - header_bad = TestNativeMSGArea.create_test_header( + header_bad = create_test_header( dataset_id=make_dataid(name='VIS006', resolution=3000), earth_model=1, is_full_disk=True, @@ -1417,11 +1209,11 @@ def test_header_warning(): fromfile.return_value = header_good with warnings.catch_warnings(): warnings.simplefilter("error") - NativeMSGFileHandler('myfile', {}, None) + NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) fromfile.return_value = header_bad with pytest.warns(UserWarning, match=exp_warning): - NativeMSGFileHandler('myfile', {}, None) + NativeMSGFileHandler(filename=None, filename_info={}, filetype_info=None) # check that without Main Header the code doesn't crash header_missing = header_good.copy() @@ -1434,27 +1226,29 @@ def test_header_warning(): @pytest.mark.parametrize( "starts_with, expected", - [(ASCII_STARTSWITH, True), - (b'invalid_startswith', False)] + [ + (ASCII_STARTSWITH, True), + (b'this_shall_fail', False) + ] ) def test_has_archive_header(starts_with, expected): """Test if the file includes an ASCII archive header.""" with mock.patch("builtins.open", mock.mock_open(read_data=starts_with)): - assert has_archive_header('filename') == expected + actual = has_archive_header('filename') + assert actual == expected def test_read_header(): """Test that reading header returns the header correctly converted to a dictionary.""" - expected = {'SatelliteId': 324, 'NominalLongitude': 0.0, 'SatelliteStatus': 1} + keys = ('SatelliteId', 'NominalLongitude', 'SatelliteStatus') + values = (324, 0.0, 1) + expected = dict(zip(keys, values)) - dtypes = np.dtype([ - ('SatelliteId', np.uint16), - ('NominalLongitude', np.float32), - ('SatelliteStatus', np.uint8) - ]) - hdr_data = np.array([(324, 0.0, 1)], dtype=dtypes) + types = (np.uint16, np.float32, np.uint8) + dtypes = np.dtype([(k, t) for k, t in zip(keys, types)]) + hdr_data = np.array([values], dtype=dtypes) with mock.patch('satpy.readers.seviri_l1b_native.np.fromfile') as fromfile: fromfile.return_value = hdr_data actual = recarray2dict(hdr_data) - unittest.TestCase().assertDictEqual(actual, expected) + assert actual == expected diff --git a/satpy/tests/reader_tests/utils.py b/satpy/tests/reader_tests/utils.py index 9415ac56ec..05e6d9cb18 100644 --- a/satpy/tests/reader_tests/utils.py +++ b/satpy/tests/reader_tests/utils.py @@ -18,6 +18,7 @@ """Utilities for reader tests.""" import inspect +import os def default_attr_processor(root, attr): @@ -61,3 +62,23 @@ def get_jit_methods(module): def _is_jit_method(obj): return hasattr(obj, "py_func") + + +def skip_numba_unstable_if_missing(): + """Determine if numba-based tests should be skipped during unstable CI tests. + + If numba fails to import it could be because numba is not compatible with + a newer version of numpy. This is very likely to happen in the + unstable/experimental CI environment. This function returns ``True`` if + numba-based tests should be skipped if ``numba`` could not + be imported *and* we're in the unstable environment. We determine if we're + in this CI environment by looking for the ``UNSTABLE="1"`` + environment variable. + + """ + try: + import numba + except ImportError: + numba = None + + return numba is None and os.environ.get("UNSTABLE", "0") in ("1", "true") diff --git a/satpy/tests/writer_tests/test_cf.py b/satpy/tests/writer_tests/test_cf.py index 2b0a5dfc6c..baeb45a4e4 100644 --- a/satpy/tests/writer_tests/test_cf.py +++ b/satpy/tests/writer_tests/test_cf.py @@ -1448,5 +1448,5 @@ def _should_use_compression_keyword(): versions = _get_backend_versions() return ( versions["libnetcdf"] >= Version("4.9.0") and - versions["xarray"] >= Version("2023.7") + versions["xarray"] >= Version("2023.8") )