From 2d34ef9b50f8b3c1c6b6933ccb771b2ba6ff2e26 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 12 Jun 2023 10:31:14 -0500 Subject: [PATCH 01/23] Switch to imageio v3 api and the pyav plugin --- continuous_integration/environment.yaml | 2 +- setup.py | 1 + uwsift/tests/view/test_export_image.py | 78 +++++++++++++++++-------- uwsift/view/export_image.py | 70 ++++++++++++---------- uwsift/view/scene_graph.py | 28 ++++++--- 5 files changed, 112 insertions(+), 67 deletions(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index a7229d27..41d61e54 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -6,7 +6,7 @@ dependencies: - defusedxml - Cython - imageio - - imageio-ffmpeg + - av - matplotlib - numba - numpy diff --git a/setup.py b/setup.py index 4a6da53b..8f080806 100644 --- a/setup.py +++ b/setup.py @@ -202,6 +202,7 @@ def run(self): "donfig", "h5py", "imageio", + "av", "matplotlib", "netCDF4", "numba", diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index dea32715..7e69eb69 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -1,8 +1,7 @@ import datetime import os -from collections import namedtuple -import imageio +import numpy as np import pytest from matplotlib import pyplot as plt from numpy.testing import assert_array_equal @@ -53,6 +52,7 @@ def __init__(self, frame_range, filename): "colorbar": True, "font_size": 10, "loop": True, + "fps": None, } def get_info(self): @@ -72,16 +72,27 @@ def get_current_frame_index(self): return max(self._frame_order, 1) if self._frame_order else 0 def get_frame_uuids(self): - return self._frame_order if self._frame_order else [] + # no need to get more UUIDs than what we're going to use in the test + return list(range(max(self._frame_order))) if self._frame_order else [] class MockSGM: + fake_screenshot_shape = (5, 10, 4) + def __init__(self): self.animation_controller = MockAnimationController() def get_screenshot_array(self, fr): if fr is None: - return None - return [[fr[1], fr[1] - fr[0]]] + return [("", np.zeros(self.fake_screenshot_shape, dtype=np.uint8))] + frames = [] + for frame_idx in range(fr[0], fr[1] + 1): + frames.append( + ( + str(frame_idx), + np.zeros(self.fake_screenshot_shape, dtype=np.uint8), + ) + ) + return frames return MockSGM() @@ -164,19 +175,39 @@ def test_convert_frame_range(range, exp, window): @pytest.mark.parametrize( - "info,isgif,exp", + "info,exp", [ - ({"fps": None, "filename": None, "loop": 0}, True, {"duration": [0.1, 0.1], "loop": 0}), - ({"fps": None, "filename": None, "loop": 0}, False, {"fps": 10}), - ({"fps": 1, "filename": None}, True, {"fps": 1, "loop": 0}), - ({"fps": 1, "filename": None}, False, {"fps": 1}), + ({"fps": None, "filename": "test.gif", "loop": 0}, {"duration": [0.1, 0.1], "loop": 0}), + ( + {"fps": None, "filename": "test.mp4", "loop": 0}, + { + "fps": 10, + "codec": "libx264", + "plugin": "pyav", + "in_pixel_format": "rgba", + "filter_sequence": [ + ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + ], + }, + ), + ({"fps": 1, "filename": "test.gif"}, {"fps": 1, "loop": 0}), + ( + {"fps": 1, "filename": "test.mp4"}, + { + "fps": 1, + "codec": "libx264", + "plugin": "pyav", + "in_pixel_format": "rgba", + "filter_sequence": [ + ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + ], + }, + ), ], ) -def test_get_animation_parameters(info, isgif, exp, monkeypatch, window): +def test_get_animation_parameters(info, exp, monkeypatch, window): """Test animation parameters are calculated correctly.""" monkeypatch.setattr(window.export_image, "model", _get_mock_model()) - monkeypatch.setattr(export_image, "is_gif_filename", lambda x: isgif) - im = Image.new("RGBA", (100, 100)) res = window.export_image._get_animation_parameters(info, [(0, im), (0, im)]) @@ -217,32 +248,29 @@ def test_create_filenames(uuids, base, exp, monkeypatch, window): [ ([1, 2], "test.gif", True, 1), ([1, 2], "test.m4v", True, 1), + ([1, 2], "test.mp4", True, 1), (None, "test.m4v", True, 0), ([1, 2], "test.gif", False, 0), ], ) -def test_save_screenshot(fr, fn, overwrite, exp, monkeypatch, window): +def test_save_screenshot(fr, fn, overwrite, exp, monkeypatch, window, tmp_path): """Test screenshot is saved correctly given the frame range and filename.""" - writer = _get_mock_writer() - IFormat = namedtuple("IFormat", "name") - + # TODO: Remove append colorbar mock? + # TODO: Only mock overwrite dialog if overwrite should be needed + # TODO: Remove footer mock + # TODO: Create the file so it can be overwritten + fn = str(tmp_path / fn) monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, fn)) - monkeypatch.setattr(window.export_image, "_convert_frame_range", lambda x: fr) monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) - monkeypatch.setattr(window.export_image, "_create_filenames", lambda x, y: ([1], [fn])) monkeypatch.setattr(window.export_image, "model", _get_mock_model()) monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) monkeypatch.setattr(window.export_image, "_append_colorbar", lambda x, y, z: y) monkeypatch.setattr(window.export_image, "_add_screenshot_footer", lambda x, y, font_size=10: x) - monkeypatch.setattr(window.export_image, "_get_animation_parameters", lambda x, y: {"loop": True}) - monkeypatch.setattr(export_image, "get_imageio_format", lambda x: IFormat(name="test")) - monkeypatch.setattr(os.path, "isfile", lambda x: True) - monkeypatch.setattr(Image, "fromarray", lambda x: x) - monkeypatch.setattr(imageio, "get_writer", lambda x, y: writer) window.export_image._save_screenshot() - assert len(writer.data) == exp + # assert len(writer.data) == exp + assert os.path.isfile(fn) def test_cmd_open_export_image_dialog(qtbot, window): diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index b932ca86..76311607 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -2,9 +2,10 @@ import logging import os -import imageio +import imageio.v3 as imageio import matplotlib as mpl import numpy +import numpy as np from matplotlib import pyplot as plt from PIL import Image, ImageDraw, ImageFont from PyQt5 import QtCore, QtGui, QtWidgets @@ -21,6 +22,14 @@ NUM_TICKS = 8 TICK_SIZE = 14 FONT = "arial" +PYAV_ANIMATION_PARAMS = { + "codec": "libx264", + "plugin": "pyav", + "in_pixel_format": "rgba", + "filter_sequence": [ + ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + ], +} def is_gif_filename(fn): @@ -31,12 +40,6 @@ def is_video_filename(fn): return os.path.splitext(fn)[-1] in [".mp4", ".m4v", ".gif"] -def get_imageio_format(fn): - """Ask imageio if it knows what to do with this filename.""" - request = imageio.core.Request(fn, "w?") - return imageio.formats.search_write_format(request) - - class ExportImageDialog(QtWidgets.QDialog): default_filename = "sift_screenshot.png" @@ -344,27 +347,35 @@ def _overwrite_dialog(self): def _get_animation_parameters(self, info, images): params = {} if info["fps"] is None: - t = [self.model.get_dataset_by_uuid(u).info.get(Info.SCHED_TIME) for u, im in images] - t_diff = [max(1, (t[i] - t[i - 1]).total_seconds()) for i in range(1, len(t))] - min_diff = float(min(t_diff)) - # imageio seems to be using duration in seconds - # so use 1/10th of a second - duration = [0.1 * (this_diff / min_diff) for this_diff in t_diff] - duration = [duration[0]] + duration - if not info["loop"]: - duration = duration + duration[-2:0:-1] - params["duration"] = duration + params["duration"] = self._get_time_lapse_duration(images, info["loop"]) else: params["fps"] = info["fps"] + is_gif = is_gif_filename(info["filename"]) if is_gif_filename(info["filename"]): params["loop"] = 0 # infinite number of loops elif "duration" in params: # not gif but were given "Time Lapse", can only have one FPS params["fps"] = int(1.0 / params.pop("duration")[0]) + if not is_gif: + params.update(PYAV_ANIMATION_PARAMS) return params + def _get_time_lapse_duration(self, images, is_loop): + if len(images) <= 1: + return [1.0] # arbitrary single frame duration + t = [self.model.get_dataset_by_uuid(u).info.get(Info.SCHED_TIME) for u, im in images] + t_diff = [max(1, (t[i] - t[i - 1]).total_seconds()) for i in range(1, len(t))] + min_diff = float(min(t_diff)) + # imageio seems to be using duration in seconds + # so use 1/10th of a second + duration = [0.1 * (this_diff / min_diff) for this_diff in t_diff] + duration = [duration[0]] + duration + if not is_loop: + duration = duration + duration[-2:0:-1] + return duration + def _convert_frame_range(self, frame_range): """Convert 1-based frame range to SGM's 0-based""" if frame_range is None: @@ -418,15 +429,7 @@ def _save_screenshot(self): for (u, im), bt in zip(images, banner_text) ] - imageio_format = get_imageio_format(filenames[0]) - if imageio_format: - format_name = imageio_format.name - elif filenames[0].upper().endswith(".M4V"): - format_name = "MP4" - else: - raise ValueError("Not sure how to handle file with format: {}".format(filenames[0])) - - if is_video_filename(filenames[0]) and len(images) > 1: + if is_video_filename(filenames[0]): params = self._get_animation_parameters(info, images) if not info["loop"] and is_gif_filename(filenames[0]): # rocking animation @@ -439,11 +442,14 @@ def _save_screenshot(self): params = {} filenames = list(zip(filenames, [[x] for x in images])) - self._write_images(filenames, format_name, params) + self._write_images(filenames, params) - def _write_images(self, filenames, format_name, params): + def _write_images(self, filenames, params): for filename, file_images in filenames: - writer = imageio.get_writer(filename, format_name, **params) - for _, x in file_images: - writer.append_data(numpy.array(x)) - writer.close() + images_arrays = [np.array(image) for _, image in file_images] + try: + imageio.imwrite(filename, images_arrays, **params) + except IOError: + msg = "Failed to write to file: {}".format(filename) + LOG.error(msg) + raise diff --git a/uwsift/view/scene_graph.py b/uwsift/view/scene_graph.py index e70bf30d..39128b4a 100644 --- a/uwsift/view/scene_graph.py +++ b/uwsift/view/scene_graph.py @@ -15,19 +15,14 @@ 3. add node instances to scene by making them children of canvas scene, or of nodes already in the scene -REFERENCES -http://api.vispy.org/en/latest/scene.html - -:author: R.K.Garcia -:copyright: 2014 by University of Wisconsin Regents, see AUTHORS for more details -:license: GPLv3, see LICENSE for more details """ +from __future__ import annotations import logging import os from enum import Enum from numbers import Number -from typing import Optional +from typing import TYPE_CHECKING, Optional from uuid import UUID import numpy as np @@ -79,6 +74,10 @@ map_point_style_to_marker_kwargs, ) +if TYPE_CHECKING: + import numpy.typing as npt + + LOG = logging.getLogger(__name__) DATA_DIR = get_package_data_dir() DEFAULT_SHAPE_FILE = os.path.join(DATA_DIR, "ne_50m_admin_0_countries", "ne_50m_admin_0_countries.shp") @@ -313,9 +312,20 @@ def __init__( self._setup_initial_canvas(center) self.pending_polygon = PendingPolygon(self.main_map) - def get_screenshot_array(self, frame_range=None): - """Get numpy arrays representing the current canvas.""" + def get_screenshot_array( + self, frame_range: None | tuple[int, int] = None + ) -> list[tuple[str | UUID, npt.NDArray[np.uint8]]]: + """Get numpy arrays representing the current canvas. + Args: + frame_range: Start and end frame indexes to get arrays for. + Indexes are 0-based and both values (start and end) are + *inclusive*. Specifying ``(1, 3)`` means you'll get arrays + for frame at index 1 (the second frame), index 2, and index 3. + If not specified or ``None`` the current frame's data is + returned. + + """ if frame_range is None: self.main_canvas.on_draw(None) return [("", _screenshot())] From e175fd718308e11e5214969450e3b3094be2fb9f Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 12 Jun 2023 11:26:47 -0500 Subject: [PATCH 02/23] Remove unnecessary mocking in export image tests --- uwsift/tests/view/test_export_image.py | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index 7e69eb69..f7f34ef4 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -49,7 +49,7 @@ def __init__(self, frame_range, filename): "frame_range": frame_range, "include_footer": True, "filename": filename, - "colorbar": True, + "colorbar": "vertical", "font_size": 10, "loop": True, "fps": None, @@ -75,11 +75,15 @@ def get_frame_uuids(self): # no need to get more UUIDs than what we're going to use in the test return list(range(max(self._frame_order))) if self._frame_order else [] + class MockCanvas: + dpi = 100 + class MockSGM: fake_screenshot_shape = (5, 10, 4) def __init__(self): self.animation_controller = MockAnimationController() + self.main_canvas = MockCanvas() def get_screenshot_array(self, fr): if fr is None: @@ -255,22 +259,17 @@ def test_create_filenames(uuids, base, exp, monkeypatch, window): ) def test_save_screenshot(fr, fn, overwrite, exp, monkeypatch, window, tmp_path): """Test screenshot is saved correctly given the frame range and filename.""" - # TODO: Remove append colorbar mock? - # TODO: Only mock overwrite dialog if overwrite should be needed - # TODO: Remove footer mock - # TODO: Create the file so it can be overwritten - fn = str(tmp_path / fn) - monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, fn)) + fn = tmp_path / fn + if overwrite: + fn.touch() + monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) + + monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(fn))) monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) monkeypatch.setattr(window.export_image, "model", _get_mock_model()) - monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) - monkeypatch.setattr(window.export_image, "_append_colorbar", lambda x, y, z: y) - monkeypatch.setattr(window.export_image, "_add_screenshot_footer", lambda x, y, font_size=10: x) window.export_image._save_screenshot() - - # assert len(writer.data) == exp - assert os.path.isfile(fn) + assert fn.is_file() def test_cmd_open_export_image_dialog(qtbot, window): From 54a1809789fcb7f033583cfaa515dadcefa044d9 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 12 Jun 2023 21:11:41 -0500 Subject: [PATCH 03/23] Add single image tests to export_image Includes workaround for JPEG with no alpha support --- uwsift/tests/view/test_export_image.py | 76 ++++++++++++++++---------- uwsift/view/export_image.py | 2 + 2 files changed, 50 insertions(+), 28 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index f7f34ef4..ecad782a 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -1,5 +1,6 @@ import datetime import os +from typing import Any import numpy as np import pytest @@ -20,22 +21,26 @@ def __init__(self): self.climits = (0, 1) class MockDataset: - def __init__(self): - self.info = {} + def __init__(self, offset: int): + self.info: dict[str, Any] = {} self.info["unit_conversion"] = ("unit", lambda t: t, lambda t: t) - self.info["timeline"] = datetime.datetime(2000, 1, 1, 0, 0, 0, 0) + self.info["timeline"] = datetime.datetime(2000, 1, 1, 0, 0, 0, 0) + datetime.timedelta(minutes=offset) self.info["display_name"] = "name" class MockModel: def __init__(self): self.moc_prez = MockPrez() - self.moc_dataset = MockDataset() + self._offset = 0 + self._offset_for_uuid = {} def get_dataset_presentation_by_uuid(self, u): return self.moc_prez def get_dataset_by_uuid(self, u): - return self.moc_dataset + self._offset_for_uuid[u] = self._offset + ds = MockDataset(self._offset) + self._offset += 1 + return ds return MockModel() @@ -101,22 +106,6 @@ def get_screenshot_array(self, fr): return MockSGM() -def _get_mock_writer(): - """Mock Writer class for testing.""" - - class MockWriter: - def __init__(self): - self.data = [] - - def append_data(self, data): - self.data.append(data) - - def close(self): - pass - - return MockWriter() - - @pytest.mark.parametrize( "size,mode,exp", [ @@ -248,16 +237,16 @@ def test_create_filenames(uuids, base, exp, monkeypatch, window): @pytest.mark.parametrize( - "fr,fn,overwrite,exp", + "fr,fn,overwrite", [ - ([1, 2], "test.gif", True, 1), - ([1, 2], "test.m4v", True, 1), - ([1, 2], "test.mp4", True, 1), - (None, "test.m4v", True, 0), - ([1, 2], "test.gif", False, 0), + ([1, 2], "test.gif", True), + ([1, 2], "test.m4v", True), + ([1, 2], "test.mp4", True), + (None, "test.m4v", True), + ([1, 2], "test.gif", False), ], ) -def test_save_screenshot(fr, fn, overwrite, exp, monkeypatch, window, tmp_path): +def test_save_screenshot_animations(fr, fn, overwrite, monkeypatch, window, tmp_path): """Test screenshot is saved correctly given the frame range and filename.""" fn = tmp_path / fn if overwrite: @@ -272,6 +261,37 @@ def test_save_screenshot(fr, fn, overwrite, exp, monkeypatch, window, tmp_path): assert fn.is_file() +@pytest.mark.parametrize( + "fr,fn,overwrite", + [ + ([1, 2], "test_{start_time:%H%M%S}.jpg", True), + ([1, 2], "test_{start_time:%H%M%S}.png", True), + ([1, 2], "test_{start_time:%H%M%S}.png", False), + (None, "test_XXXXXX.png", True), + (None, "test_XXXXXX.png", False), + ], +) +def test_save_screenshot_images(fr, fn, overwrite, monkeypatch, window, tmp_path): + """Test screenshot is saved correctly given the frame range and filename.""" + exp_num = 1 if fr is None else len(fr) + expected_files = [ + tmp_path / fn.format(start_time=datetime.datetime(2000, 1, 1, 0, offset, 0)) for offset in range(exp_num) + ] + if overwrite: + for exp_file in expected_files: + exp_file.touch() + monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) + + monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(tmp_path / fn))) + monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) + monkeypatch.setattr(window.export_image, "model", _get_mock_model()) + + window.export_image._save_screenshot() + assert len(list(tmp_path.iterdir())) == len(expected_files) + for exp_file in expected_files: + assert exp_file.is_file() + + def test_cmd_open_export_image_dialog(qtbot, window): """Test that the keyboard shortcut Ctrl/Cmd + I opens the export image menu.""" qtbot.addWidget(window) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index 76311607..235cc817 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -447,6 +447,8 @@ def _save_screenshot(self): def _write_images(self, filenames, params): for filename, file_images in filenames: images_arrays = [np.array(image) for _, image in file_images] + if filename.endswith(".jpg"): + images_arrays = [image_arr[:, :, :3] for image_arr in images_arrays] try: imageio.imwrite(filename, images_arrays, **params) except IOError: From 3a18125d70485c54042b041aa8e68850c927c9b9 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 13 Jun 2023 10:02:06 -0500 Subject: [PATCH 04/23] Update JPEG RGBA check for .jpg and .jpeg --- uwsift/view/export_image.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index 235cc817..53d1d4a9 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -4,8 +4,8 @@ import imageio.v3 as imageio import matplotlib as mpl -import numpy import numpy as np +import numpy.typing as npt from matplotlib import pyplot as plt from PIL import Image, ImageDraw, ImageFont from PyQt5 import QtCore, QtGui, QtWidgets @@ -243,7 +243,7 @@ def _create_colorbar(self, mode, u, size): colors = COLORMAP_MANAGER[colormap] if colormap == "Square Root (Vis Default)": - colors = colors.map(numpy.linspace((0, 0, 0, 1), (1, 1, 1, 1), 256)) + colors = colors.map(np.linspace((0, 0, 0, 1), (1, 1, 1, 1), 256)) else: colors = colors.colors.rgba @@ -266,9 +266,9 @@ def _create_colorbar(self, mode, u, size): self.model.get_dataset_by_uuid(u).info.get(Info.UNIT_CONVERSION)[1](t) ) ) - for t in numpy.linspace(vmin, vmax, NUM_TICKS) + for t in np.linspace(vmin, vmax, NUM_TICKS) ] - cbar.set_ticks(numpy.linspace(vmin, vmax, NUM_TICKS)) + cbar.set_ticks(np.linspace(vmin, vmax, NUM_TICKS)) cbar.set_ticklabels(ticks) return fig @@ -446,12 +446,21 @@ def _save_screenshot(self): def _write_images(self, filenames, params): for filename, file_images in filenames: - images_arrays = [np.array(image) for _, image in file_images] - if filename.endswith(".jpg"): - images_arrays = [image_arr[:, :, :3] for image_arr in images_arrays] + images_arrays = _image_to_frame_array(file_images, filename) try: imageio.imwrite(filename, images_arrays, **params) except IOError: msg = "Failed to write to file: {}".format(filename) LOG.error(msg) raise + + +def _image_to_frame_array(file_images: list[Image], filename: str) -> list[npt.NDArray[np.uint8]]: + images_arrays = [np.array(image) for _, image in file_images] + if not _supports_rgba(filename): + images_arrays = [image_arr[:, :, :3] for image_arr in images_arrays] + return images_arrays + + +def _supports_rgba(filename: str) -> bool: + return not (filename.endswith(".jpeg") or filename.endswith(".jpg")) From 1ec3682336edd0fee4c8c6000eb8480baf74c4b0 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 13 Jun 2023 11:39:18 -0500 Subject: [PATCH 05/23] Add missing future annotation import --- uwsift/view/export_image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index 53d1d4a9..bd8b2636 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import io import logging import os From 52342367fde64fdd1321f8cf5e4fbcb2bce31930 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 14 Jun 2023 12:55:28 -0500 Subject: [PATCH 06/23] Fix matplotlib figures not being closed on export image colorbar creation --- uwsift/view/export_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index bd8b2636..d7d4a70f 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -287,6 +287,7 @@ def _append_colorbar(self, mode, im, u): buf = io.BytesIO() fig.savefig(buf, format="png", bbox_inches="tight", dpi=self.sgm.main_canvas.dpi) + plt.close(fig) buf.seek(0) fig_im = Image.open(buf) @@ -300,13 +301,12 @@ def _append_colorbar(self, mode, im, u): for i in [im, fig_im]: new_im.paste(i, (offset, 0)) offset += i.size[0] - return new_im else: new_im = Image.new(im.mode, (orig_w, orig_h + fig_h)) for i in [im, fig_im]: new_im.paste(i, (0, offset)) offset += i.size[1] - return new_im + return new_im def _create_filenames(self, uuids, base_filename): if not uuids or uuids[0] is None: From 756d123f4210a0d5df1a3542bc63bc1737946d32 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 14 Jun 2023 14:07:08 -0500 Subject: [PATCH 07/23] Fix PIL deprecation warning for textsize usage --- uwsift/view/export_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index d7d4a70f..4146a5a3 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -232,7 +232,7 @@ def _add_screenshot_footer(self, im, banner_text, font_size=11): # give one extra pixel on the left to make sure letters # don't get cut off new_draw.text([1, orig_h], banner_text, fill="#ffffff", font=font) - txt_w, txt_h = new_draw.textsize("SIFT", font) + txt_w = new_draw.textlength("SIFT", font) new_draw.text([orig_w - txt_w, orig_h], "SIFT", fill="#ffffff", font=font) new_im.paste(im, (0, 0, orig_w, orig_h)) return new_im From a8c8c5e7f8149bef180e97ff6fb34a7545dd880d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 14 Jun 2023 20:59:15 -0500 Subject: [PATCH 08/23] Fix PIL deprecation and fps to pyav --- uwsift/tests/view/test_export_image.py | 19 ++++++++++++------- uwsift/view/export_image.py | 17 +++++++++++------ 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index ecad782a..8687d655 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -1,6 +1,6 @@ import datetime import os -from typing import Any +from typing import Any, Optional import numpy as np import pytest @@ -45,11 +45,15 @@ def get_dataset_by_uuid(self, u): return MockModel() -def _get_mock_sd(fr, fn): +def _get_mock_sd( + frame_range: Optional[list[int]], + filename: str, + fps: Optional[float] = None, +): """Mock ScreenshotDialog class for testing.""" class MockScreenshotDialog: - def __init__(self, frame_range, filename): + def __init__(self): self.info = { "frame_range": frame_range, "include_footer": True, @@ -57,13 +61,13 @@ def __init__(self, frame_range, filename): "colorbar": "vertical", "font_size": 10, "loop": True, - "fps": None, + "fps": fps, } def get_info(self): return self.info - return MockScreenshotDialog(fr, fn) + return MockScreenshotDialog() def _get_mock_sgm(frame_order): @@ -246,14 +250,15 @@ def test_create_filenames(uuids, base, exp, monkeypatch, window): ([1, 2], "test.gif", False), ], ) -def test_save_screenshot_animations(fr, fn, overwrite, monkeypatch, window, tmp_path): +@pytest.mark.parametrize("fps", [None, 2.2]) +def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, tmp_path): """Test screenshot is saved correctly given the frame range and filename.""" fn = tmp_path / fn if overwrite: fn.touch() monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) - monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(fn))) + monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(fn), fps)) monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) monkeypatch.setattr(window.export_image, "model", _get_mock_model()) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index 4146a5a3..a6b2b82e 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -3,6 +3,7 @@ import io import logging import os +from fractions import Fraction import imageio.v3 as imageio import matplotlib as mpl @@ -354,14 +355,17 @@ def _get_animation_parameters(self, info, images): params["fps"] = info["fps"] is_gif = is_gif_filename(info["filename"]) - if is_gif_filename(info["filename"]): + if is_gif: params["loop"] = 0 # infinite number of loops - elif "duration" in params: - # not gif but were given "Time Lapse", can only have one FPS - params["fps"] = int(1.0 / params.pop("duration")[0]) - - if not is_gif: + if "fps" in params: + # PIL duration in milliseconds + params["duration"] = [1.0 / params.pop("fps") * 1000.0] * len(images) + else: + if "duration" in params: + # not gif but were given "Time Lapse", can only have one FPS + params["fps"] = 1.0 / params.pop("duration")[0] params.update(PYAV_ANIMATION_PARAMS) + params["fps"] = Fraction(params["fps"]).limit_denominator(65535) return params def _get_time_lapse_duration(self, images, is_loop): @@ -448,6 +452,7 @@ def _save_screenshot(self): def _write_images(self, filenames, params): for filename, file_images in filenames: + print(filename, file_images) images_arrays = _image_to_frame_array(file_images, filename) try: imageio.imwrite(filename, images_arrays, **params) From 90193aa9c08cae742e8c3570ad176834e4c64ae6 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 15 Jun 2023 06:53:36 -0500 Subject: [PATCH 09/23] Fix export image expected kwargs test --- uwsift/tests/view/test_export_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index 8687d655..c4bd0ffe 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -187,7 +187,7 @@ def test_convert_frame_range(range, exp, window): ], }, ), - ({"fps": 1, "filename": "test.gif"}, {"fps": 1, "loop": 0}), + ({"fps": 1, "filename": "test.gif"}, {"duration": [1000.0, 1000.0], "loop": 0}), ( {"fps": 1, "filename": "test.mp4"}, { From abb61fcd18b49b261d0e0f4f3758efcfca59a931 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 15 Jun 2023 08:11:40 -0500 Subject: [PATCH 10/23] Add missing annotations import --- uwsift/tests/view/test_export_image.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index c4bd0ffe..759f9e6d 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import os from typing import Any, Optional From 5572d86b299c79192ffda4c5a96916b8947ae22e Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 15 Jun 2023 11:38:09 -0500 Subject: [PATCH 11/23] Fix animation scaling being too aggressive --- uwsift/tests/view/test_export_image.py | 19 +++++++++++++++---- uwsift/view/export_image.py | 4 ++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index 759f9e6d..84959228 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -4,6 +4,7 @@ import os from typing import Any, Optional +import imageio.v3 as imageio import numpy as np import pytest from matplotlib import pyplot as plt @@ -95,16 +96,17 @@ class MockSGM: def __init__(self): self.animation_controller = MockAnimationController() self.main_canvas = MockCanvas() + self.rng = np.random.default_rng() def get_screenshot_array(self, fr): if fr is None: - return [("", np.zeros(self.fake_screenshot_shape, dtype=np.uint8))] + return [("", self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8))] frames = [] for frame_idx in range(fr[0], fr[1] + 1): frames.append( ( str(frame_idx), - np.zeros(self.fake_screenshot_shape, dtype=np.uint8), + self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8), ) ) return frames @@ -185,7 +187,7 @@ def test_convert_frame_range(range, exp, window): "plugin": "pyav", "in_pixel_format": "rgba", "filter_sequence": [ - ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), ], }, ), @@ -198,7 +200,7 @@ def test_convert_frame_range(range, exp, window): "plugin": "pyav", "in_pixel_format": "rgba", "filter_sequence": [ - ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), ], }, ), @@ -266,6 +268,15 @@ def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, window.export_image._save_screenshot() assert fn.is_file() + if fn.suffix == ".gif": + exp_frame_shape = (15, 20) + read_kwargs = {"plugin": "pillow", "mode": "RGBA"} + else: + exp_frame_shape = (480, 640) + read_kwargs = {"plugin": "pyav"} + frames = imageio.imread(fn, **read_kwargs) + assert frames.shape[0] == (len(fr) if fr is not None else 1) + assert frames[0].shape[:2] == exp_frame_shape @pytest.mark.parametrize( diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index a6b2b82e..e3ebce1e 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -30,7 +30,8 @@ "plugin": "pyav", "in_pixel_format": "rgba", "filter_sequence": [ - ("scale", "trunc(iw/2)*2:trunc(ih/2)*2"), + # Scale animation frames to the macro buffer size (16) + ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), ], } @@ -452,7 +453,6 @@ def _save_screenshot(self): def _write_images(self, filenames, params): for filename, file_images in filenames: - print(filename, file_images) images_arrays = _image_to_frame_array(file_images, filename) try: imageio.imwrite(filename, images_arrays, **params) From 67092f1476f5f389e24e6e4529c59e1d5d6ccf36 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 19 Jun 2023 09:36:12 -0500 Subject: [PATCH 12/23] Update export image testing for no loaded data cases --- uwsift/tests/view/test_export_image.py | 106 +++++++++++++++---------- uwsift/view/export_image.py | 2 +- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index 84959228..3c01e3ca 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -12,6 +12,7 @@ from PIL import Image from PyQt5.QtCore import Qt +from uwsift.common import Info from uwsift.view import export_image @@ -25,10 +26,10 @@ def __init__(self): class MockDataset: def __init__(self, offset: int): - self.info: dict[str, Any] = {} - self.info["unit_conversion"] = ("unit", lambda t: t, lambda t: t) - self.info["timeline"] = datetime.datetime(2000, 1, 1, 0, 0, 0, 0) + datetime.timedelta(minutes=offset) - self.info["display_name"] = "name" + self.info: dict[Info, Any] = {} + self.info[Info.UNIT_CONVERSION] = ("unit", lambda t: t, lambda t: t) + self.info[Info.SCHED_TIME] = datetime.datetime(2000, 1, 1, 0, 0, 0, 0) + datetime.timedelta(minutes=offset) + self.info[Info.DISPLAY_NAME] = "name" class MockModel: def __init__(self): @@ -75,43 +76,57 @@ def get_info(self): def _get_mock_sgm(frame_order): """Mock SceneGraphManager class for testing.""" - - class MockAnimationController: - def __init__(self): - self._frame_order = frame_order - - def get_current_frame_index(self): - return max(self._frame_order, 1) if self._frame_order else 0 - - def get_frame_uuids(self): - # no need to get more UUIDs than what we're going to use in the test - return list(range(max(self._frame_order))) if self._frame_order else [] - - class MockCanvas: - dpi = 100 - - class MockSGM: - fake_screenshot_shape = (5, 10, 4) - - def __init__(self): - self.animation_controller = MockAnimationController() - self.main_canvas = MockCanvas() - self.rng = np.random.default_rng() - - def get_screenshot_array(self, fr): - if fr is None: - return [("", self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8))] - frames = [] - for frame_idx in range(fr[0], fr[1] + 1): - frames.append( - ( - str(frame_idx), - self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8), - ) + return _MockSGM(frame_order) + + +class _MockAnimationController: + def __init__(self, frame_order): + self._frame_order = frame_order + + def get_current_frame_index(self): + # frame range of `False` (in these unit tests) means no data is loaded + return max(self._frame_order or 1, 1) if self._frame_order is not False else 0 + + def get_frame_uuids(self): + if self._frame_order is False: + # no data loaded, so no UUIDs + return [] + if self._frame_order is None: + # single "current layer" being shown + # start fake UUIDs at 1 to avoid `if not uuid:` failures with 0 + return [1] + # no need to get more UUIDs than what we're going to use in the test + return list(range(1, max(self._frame_order) + 1)) + + +class _MockCanvas: + dpi = 100 + + +class _MockSGM: + fake_screenshot_shape = (5, 10, 4) + + def __init__(self, frame_order): + self.animation_controller = _MockAnimationController(frame_order) + self.main_canvas = _MockCanvas() + self.rng = np.random.default_rng() + self._frame_order = frame_order + + def get_screenshot_array(self, fr): + if self._frame_order is False and fr is None: + # no data loaded + return [("", self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8))] + if fr is None: + fr = (1, 1) + frames = [] + for frame_idx in range(fr[0], fr[1] + 1): + frames.append( + ( + str(frame_idx), + self.rng.integers(0, 255, self.fake_screenshot_shape, dtype=np.uint8), ) - return frames - - return MockSGM() + ) + return frames @pytest.mark.parametrize( @@ -262,7 +277,7 @@ def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, fn.touch() monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) - monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(fn), fps)) + monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr or None, str(fn), fps)) monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) monkeypatch.setattr(window.export_image, "model", _get_mock_model()) @@ -275,7 +290,7 @@ def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, exp_frame_shape = (480, 640) read_kwargs = {"plugin": "pyav"} frames = imageio.imread(fn, **read_kwargs) - assert frames.shape[0] == (len(fr) if fr is not None else 1) + assert frames.shape[0] == (len(fr) if fr else 1) assert frames[0].shape[:2] == exp_frame_shape @@ -287,11 +302,12 @@ def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, ([1, 2], "test_{start_time:%H%M%S}.png", False), (None, "test_XXXXXX.png", True), (None, "test_XXXXXX.png", False), + (False, "test_XXXXXX.png", False), ], ) def test_save_screenshot_images(fr, fn, overwrite, monkeypatch, window, tmp_path): """Test screenshot is saved correctly given the frame range and filename.""" - exp_num = 1 if fr is None else len(fr) + exp_num = 1 if not fr else len(fr) expected_files = [ tmp_path / fn.format(start_time=datetime.datetime(2000, 1, 1, 0, offset, 0)) for offset in range(exp_num) ] @@ -300,7 +316,7 @@ def test_save_screenshot_images(fr, fn, overwrite, monkeypatch, window, tmp_path exp_file.touch() monkeypatch.setattr(window.export_image, "_overwrite_dialog", lambda: overwrite) - monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr, str(tmp_path / fn))) + monkeypatch.setattr(window.export_image, "_screenshot_dialog", _get_mock_sd(fr or None, str(tmp_path / fn))) monkeypatch.setattr(window.export_image, "sgm", _get_mock_sgm(fr)) monkeypatch.setattr(window.export_image, "model", _get_mock_model()) @@ -308,6 +324,8 @@ def test_save_screenshot_images(fr, fn, overwrite, monkeypatch, window, tmp_path assert len(list(tmp_path.iterdir())) == len(expected_files) for exp_file in expected_files: assert exp_file.is_file() + img = imageio.imread(exp_file) + assert img.shape[:2] == (15, 20) def test_cmd_open_export_image_dialog(qtbot, window): diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index e3ebce1e..934979cb 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -30,7 +30,7 @@ "plugin": "pyav", "in_pixel_format": "rgba", "filter_sequence": [ - # Scale animation frames to the macro buffer size (16) + # Scale animation frames to the macro block size (16) ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), ], } From 4eeddcb4eb7bc279a797909f07f7a03d6d5c475a Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 25 Jul 2023 10:17:54 -0500 Subject: [PATCH 13/23] Fix export image when no frame range if provided --- uwsift/view/scene_graph.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/uwsift/view/scene_graph.py b/uwsift/view/scene_graph.py index 39128b4a..bd8afea5 100644 --- a/uwsift/view/scene_graph.py +++ b/uwsift/view/scene_graph.py @@ -326,13 +326,18 @@ def get_screenshot_array( returned. """ - if frame_range is None: + # Store current index to reset the view once we are done + # Or use it as the frame to screenshot if no frame range is specified + current_frame = self.animation_controller.get_current_frame_index() + if not current_frame: + # no data loaded self.main_canvas.on_draw(None) return [("", _screenshot())] + if frame_range is None: + # screenshot the current view + s = e = current_frame else: s, e = frame_range - # Store current index to reset the view once we are done - current_frame = self.animation_controller.get_current_frame_index() images = [] for i in range(s, e + 1): From ff9e4a1a2b0e47f50291f45338c186b9405fc6cf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 22 Aug 2023 21:27:17 -0500 Subject: [PATCH 14/23] Fix handling of animation frames that aren't factor of 2 --- uwsift/tests/view/test_export_image.py | 8 +------- uwsift/view/export_image.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/uwsift/tests/view/test_export_image.py b/uwsift/tests/view/test_export_image.py index 3c01e3ca..eead72ff 100644 --- a/uwsift/tests/view/test_export_image.py +++ b/uwsift/tests/view/test_export_image.py @@ -201,9 +201,6 @@ def test_convert_frame_range(range, exp, window): "codec": "libx264", "plugin": "pyav", "in_pixel_format": "rgba", - "filter_sequence": [ - ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), - ], }, ), ({"fps": 1, "filename": "test.gif"}, {"duration": [1000.0, 1000.0], "loop": 0}), @@ -214,9 +211,6 @@ def test_convert_frame_range(range, exp, window): "codec": "libx264", "plugin": "pyav", "in_pixel_format": "rgba", - "filter_sequence": [ - ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), - ], }, ), ], @@ -287,7 +281,7 @@ def test_save_screenshot_animations(fr, fn, overwrite, fps, monkeypatch, window, exp_frame_shape = (15, 20) read_kwargs = {"plugin": "pillow", "mode": "RGBA"} else: - exp_frame_shape = (480, 640) + exp_frame_shape = (14, 20) read_kwargs = {"plugin": "pyav"} frames = imageio.imread(fn, **read_kwargs) assert frames.shape[0] == (len(fr) if fr else 1) diff --git a/uwsift/view/export_image.py b/uwsift/view/export_image.py index 934979cb..e68cf955 100644 --- a/uwsift/view/export_image.py +++ b/uwsift/view/export_image.py @@ -29,10 +29,6 @@ "codec": "libx264", "plugin": "pyav", "in_pixel_format": "rgba", - "filter_sequence": [ - # Scale animation frames to the macro block size (16) - ("scale", "iw+gt(mod(iw,16), 0)*(16-mod(iw,16)):ih+gt(mod(ih,16), 0)*(16-mod(ih,16))"), - ], } @@ -466,8 +462,19 @@ def _image_to_frame_array(file_images: list[Image], filename: str) -> list[npt.N images_arrays = [np.array(image) for _, image in file_images] if not _supports_rgba(filename): images_arrays = [image_arr[:, :, :3] for image_arr in images_arrays] + # make sure frames are divisible by 2 to make ffmpeg happy + if is_video_filename(filename) and not is_gif_filename(filename): + images_arrays = [_array_divisible_by_2(img_array) for img_array in images_arrays] return images_arrays +def _array_divisible_by_2(img_array: npt.NDArray[np.uint8]) -> npt.NDArray[np.uint8]: + shape = img_array.shape + shape_by_2 = tuple(dim_size - dim_size % 2 for dim_size in shape) + if shape_by_2 == shape: + return img_array + return img_array[: shape_by_2[0], : shape_by_2[1], :] + + def _supports_rgba(filename: str) -> bool: return not (filename.endswith(".jpeg") or filename.endswith(".jpg")) From 6b4f00d5cce57c7777fdbeac050adaf5aefd247d Mon Sep 17 00:00:00 2001 From: andream Date: Wed, 28 Aug 2024 18:02:09 +0200 Subject: [PATCH 15/23] fix li filename pattern --- uwsift/etc/SIFT/config/readers/li_l1b_nc.yaml | 2 +- uwsift/etc/SIFT/config/readers/li_l2_nc.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/uwsift/etc/SIFT/config/readers/li_l1b_nc.yaml b/uwsift/etc/SIFT/config/readers/li_l1b_nc.yaml index 81966991..a1eaf2b8 100644 --- a/uwsift/etc/SIFT/config/readers/li_l1b_nc.yaml +++ b/uwsift/etc/SIFT/config/readers/li_l1b_nc.yaml @@ -3,7 +3,7 @@ data_reading: group_keys: ['start_time'] filter_patterns: [ - '{pflag}_{location_indicator},{data_designator},MTGI{spacecraft_id}+LI-1B-{type}--{subtype}--{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' + '{pflag}_{location_indicator},{data_designator},{spacecraft_id}+LI-1B-{type}--{subtype}--{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] kind: 'DYNAMIC' style_attributes: diff --git a/uwsift/etc/SIFT/config/readers/li_l2_nc.yaml b/uwsift/etc/SIFT/config/readers/li_l2_nc.yaml index 66102304..847cba8a 100644 --- a/uwsift/etc/SIFT/config/readers/li_l2_nc.yaml +++ b/uwsift/etc/SIFT/config/readers/li_l2_nc.yaml @@ -3,7 +3,7 @@ data_reading: group_keys: ['start_time'] filter_patterns: [ - '{pflag}_{location_indicator},{data_designator},MTGI{spacecraft_id}+LI-2-{type}--{subtype}--{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' + '{pflag}_{location_indicator},{data_designator},{spacecraft_id}+LI-2-{type}--{subtype}--{component1}-{component2}-{component3}-{purpose}-{format}_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.nc' ] kind: 'DYNAMIC' style_attributes: From 4f903043f2a92b0c1f3e60257e8d8e4396fd4bf7 Mon Sep 17 00:00:00 2001 From: andream Date: Wed, 28 Aug 2024 18:03:52 +0200 Subject: [PATCH 16/23] add fci l2 bufr/grib readers to list --- uwsift/etc/SIFT/config/limit_available_readers.yaml | 2 ++ uwsift/etc/SIFT/config/readers/fci_l2_bufr.yaml | 5 +++++ uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml | 4 ++++ 3 files changed, 11 insertions(+) create mode 100644 uwsift/etc/SIFT/config/readers/fci_l2_bufr.yaml create mode 100644 uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml diff --git a/uwsift/etc/SIFT/config/limit_available_readers.yaml b/uwsift/etc/SIFT/config/limit_available_readers.yaml index 8d19fe33..db4db108 100644 --- a/uwsift/etc/SIFT/config/limit_available_readers.yaml +++ b/uwsift/etc/SIFT/config/limit_available_readers.yaml @@ -9,6 +9,8 @@ data_reading: - fci_l1c_nc - fci_l1c_iqt_fdhsi - fci_l2_nc + - fci_l2_bufr + - fci_l2_grib - li_l1b_nc - li_l1b_bck_nc_sift - li_l2_nc diff --git a/uwsift/etc/SIFT/config/readers/fci_l2_bufr.yaml b/uwsift/etc/SIFT/config/readers/fci_l2_bufr.yaml new file mode 100644 index 00000000..d0dde8a1 --- /dev/null +++ b/uwsift/etc/SIFT/config/readers/fci_l2_bufr.yaml @@ -0,0 +1,5 @@ +data_reading: + fci_l2_bufr: + filter_patterns: ['{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-{type}-{subtype}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-BUFR_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.bin'] + reader_kwargs: + with_area_definition: True diff --git a/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml b/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml new file mode 100644 index 00000000..1b087bb6 --- /dev/null +++ b/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml @@ -0,0 +1,4 @@ +data_reading: + fci_l2_grib: + filter_patterns: ['{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-{type}-{subtype}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-GRIB2_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.bin'] + From 9c2aea07d9f4b1cff3e6dba2e0e1a4024d2b90a0 Mon Sep 17 00:00:00 2001 From: andream Date: Wed, 28 Aug 2024 18:04:37 +0200 Subject: [PATCH 17/23] remove extra line --- uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml b/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml index 1b087bb6..0e6eab0b 100644 --- a/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml +++ b/uwsift/etc/SIFT/config/readers/fci_l2_grib.yaml @@ -1,4 +1,3 @@ data_reading: fci_l2_grib: filter_patterns: ['{pflag}_{location_indicator},{data_designator},MTI{spacecraft_id:1d}+FCI-2-{type}-{subtype}-{coverage}-{subsetting}-{component1}-{component2}-{component3}-{purpose}-GRIB2_{oflag}_{originator}_{processing_time:%Y%m%d%H%M%S}_{facility_or_tool}_{environment}_{start_time:%Y%m%d%H%M%S}_{end_time:%Y%m%d%H%M%S}_{processing_mode}_{special_compression}_{disposition_mode}_{repeat_cycle_in_day:>04d}_{count_in_repeat_cycle:>04d}.bin'] - From c5fe1b359b0cb741840c6dfe0eaa2c0820deadb0 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 29 Aug 2024 09:45:53 +0200 Subject: [PATCH 18/23] switch from deprecated types-pkg-resources to types-setuptools --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8eec9083..c42eeafe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,7 +40,7 @@ repos: - id: mypy additional_dependencies: - types-docutils - - types-pkg-resources + - types-setuptools - types-PyYAML - types-requests - types-python-dateutil From aca9f5c126cab8dd5708ef178e4cb4fae041c2fc Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 29 Aug 2024 10:08:42 +0200 Subject: [PATCH 19/23] limit to py<=3.11 due to ecmwflibs --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d2521714..c1344450 100644 --- a/setup.py +++ b/setup.py @@ -226,7 +226,7 @@ def run(self): "cfgrib", ], tests_requires=["pytest", "pytest-qt", "pytest-mock"], - python_requires=">=3.8", + python_requires=">=3.8, <=3.11", # limiting to 3.11 until ecmwflibs is not available for 3.12 extras_require=extras_require, packages=find_packages(), entry_points={ From d873212740f94d0ea006f8dd48c63f08de07bb12 Mon Sep 17 00:00:00 2001 From: andream Date: Thu, 29 Aug 2024 10:15:01 +0200 Subject: [PATCH 20/23] limit to py<=3.11 due to ecmwflibs also in conda env yaml --- continuous_integration/environment.yaml | 2 ++ continuous_integration/rtd_environment.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index a7229d27..7934acaf 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -2,6 +2,8 @@ name: test-environment channels: - conda-forge dependencies: + # limiting python due to ecmwflibs + - python<=3.11 - appdirs - defusedxml - Cython diff --git a/continuous_integration/rtd_environment.yaml b/continuous_integration/rtd_environment.yaml index d57b0af9..68730bbf 100644 --- a/continuous_integration/rtd_environment.yaml +++ b/continuous_integration/rtd_environment.yaml @@ -2,6 +2,8 @@ name: test-environment channels: - conda-forge dependencies: + # limiting python due to ecmwflibs + - python<=3.11 - appdirs - defusedxml - Cython From 61edf41797f81e844c105bd0a685bf95a76052c8 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 29 Aug 2024 09:03:02 -0500 Subject: [PATCH 21/23] Remove zarr from experimental CI --- .github/workflows/ci.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index cc7a3963..f89a5ffe 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -128,7 +128,6 @@ jobs: --no-deps --upgrade \ git+https://github.com/dask/dask \ git+https://github.com/dask/distributed \ - git+https://github.com/zarr-developers/zarr \ git+https://github.com/Unidata/cftime \ git+https://github.com/mapbox/rasterio \ git+https://github.com/pydata/bottleneck \ From 97981a2c15847d2c88559501a198eb45632a1b62 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 30 Aug 2024 16:53:35 +0200 Subject: [PATCH 22/23] add uuid check to assess if no data is loaded --- uwsift/view/scene_graph.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/uwsift/view/scene_graph.py b/uwsift/view/scene_graph.py index bd8afea5..da0cbeb9 100644 --- a/uwsift/view/scene_graph.py +++ b/uwsift/view/scene_graph.py @@ -329,7 +329,8 @@ def get_screenshot_array( # Store current index to reset the view once we are done # Or use it as the frame to screenshot if no frame range is specified current_frame = self.animation_controller.get_current_frame_index() - if not current_frame: + current_uuid = self.animation_controller.get_current_frame_uuid() + if not current_frame and not current_uuid: # no data loaded self.main_canvas.on_draw(None) return [("", _screenshot())] From a8ae8e7735288f9703f6f4aad87c173edcb20935 Mon Sep 17 00:00:00 2001 From: andream Date: Fri, 30 Aug 2024 17:33:29 +0200 Subject: [PATCH 23/23] update ci runs python versions --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f89a5ffe..66f6eb0d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,7 +43,7 @@ jobs: with: miniforge-variant: Mambaforge miniforge-version: latest - python-version: "3.10" + python-version: "3.11" use-mamba: true environment-file: continuous_integration/environment.yaml activate-environment: test-environment @@ -73,10 +73,10 @@ jobs: # XXX: We don't currently have OpenGL installation on other platforms #os: ["windows-latest", "ubuntu-latest", "macos-latest"] os: ["ubuntu-latest"] - python-version: ["3.8", "3.10"] + python-version: ["3.9", "3.11"] experimental: [false] include: - - python-version: "3.10" + - python-version: "3.11" os: "ubuntu-latest" experimental: true