From 18c06e35663cba24e37f1c3253acc9515b61e309 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 20:27:57 -0700
Subject: [PATCH 01/29] Add serialization to dict at the video object level

---
 sleap_io/io/video.py    | 50 +++++++++++++++++++++++++++++++++++++++++
 sleap_io/model/video.py |  7 ++++++
 2 files changed, 57 insertions(+)

diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index b121c223..8f2b5b9f 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -434,6 +434,20 @@ def _read_frames(self, frame_inds: list) -> np.ndarray:
                     )
         return imgs
 
+    def to_json(self) -> dict:
+        """Return JSON-serializable dictionary of the video backend."""
+        return {
+            "type": "MediaVideo",
+            "shape": self.shape,
+            "backend": {
+                "filename": self.filename,
+                "grayscale": self.grayscale,
+                "bgr": True,
+                "dataset": "",
+                "input_format": "",
+            },
+        }
+
 
 @attrs.define
 class HDF5Video(VideoBackend):
@@ -682,6 +696,20 @@ def _read_frames(self, frame_inds: list) -> np.ndarray:
 
         return imgs
 
+    def to_json(self) -> dict:
+        """Return JSON-serializable dictionary of the video backend."""
+        return {
+            "type": "HDF5Video",
+            "shape": self.shape,
+            "backend": {
+                "filename": ("." if self.has_embedded_images else self.filename),
+                "dataset": self.dataset,
+                "input_format": self.input_format,
+                "convert_range": False,
+                "has_embedded_images": self.has_embedded_images,
+            },
+        }
+
 
 @attrs.define
 class ImageVideo(VideoBackend):
@@ -726,3 +754,25 @@ def _read_frame(self, frame_idx: int) -> np.ndarray:
         if img.ndim == 2:
             img = np.expand_dims(img, axis=-1)
         return img
+
+    def to_json(self) -> dict:
+        """Return JSON-serializable dictionary of the video backend."""
+
+        shape = self.shape
+        if shape is None:
+            height, width, channels = 0, 0, 1
+        else:
+            height, width, channels = shape[1:]
+
+        return {
+            "type": "ImageVideo",
+            "shape": self.shape,
+            "backend": {
+                "filename": self.filename[0],
+                "filenames": self.filename,
+                "height_": height,
+                "width_": width,
+                "channels_": channels,
+                "grayscale": self.grayscale,
+            },
+        }
diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index fe9c70cf..46d81473 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -231,3 +231,10 @@ def replace_filename(
                 self.open()
             else:
                 self.close()
+
+    def to_json(self) -> dict:
+        """Return a dictionary representation of the video."""
+        if self.backend is not None:
+            return self.backend.to_json()
+        else:
+            return {"filename": self.filename, "backend": None}

From 6bc26c41a85107f22b594e14303541a6747d691a Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 21:14:53 -0700
Subject: [PATCH 02/29] Store and retrieve video backend metadata

---
 sleap_io/io/slp.py      | 282 +++++++++++++++++++++++++++-------------
 sleap_io/io/video.py    |  55 +-------
 sleap_io/model/video.py |  34 ++++-
 tests/io/test_slp.py    |  23 +++-
 4 files changed, 241 insertions(+), 153 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 1f172fe4..cd5cbd12 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -20,15 +20,12 @@
     LabeledFrame,
     Labels,
 )
-from sleap_io.io.video import ImageVideo, MediaVideo, HDF5Video
+from sleap_io.io.video import VideoBackend, ImageVideo, MediaVideo, HDF5Video
 from sleap_io.io.utils import (
     read_hdf5_attrs,
     read_hdf5_dataset,
-    write_hdf5_dataset,
-    write_hdf5_group,
-    write_hdf5_attrs,
 )
-from sleap_io.io.video import VideoBackend
+import imageio.v3 as iio
 from enum import IntEnum
 from pathlib import Path
 
@@ -40,6 +37,54 @@ class InstanceType(IntEnum):
     PREDICTED = 1
 
 
+def make_video(video_json: dict, labels_path: str) -> Video:
+    """Create a `Video` object from a JSON dictionary.
+
+    Args:
+        video_json: A dictionary containing the video metadata.
+        labels_path: A string path to the SLEAP labels file.
+    """
+    backend_metadata = video_json["backend"]
+    video_path = backend_metadata["filename"]
+
+    # Marker for embedded videos.
+    if video_path == ".":
+        video_path = labels_path
+
+    # Basic path resolution.
+    video_path = Path(video_path)
+    if not video_path.exists():
+        # Check for the same filename in the same directory as the labels file.
+        video_path_ = Path(labels_path).parent / video_path.name
+        if video_path_.exists():
+            video_path = video_path_
+        else:
+            # TODO (TP): Expand capabilities of path resolution to support more
+            # complex path finding strategies.
+            pass
+
+    video_path = video_path.as_posix()
+
+    if "filenames" in backend_metadata:
+        # This is an ImageVideo.
+        # TODO: Path resolution.
+        video_path = backend_metadata["filenames"]
+
+    try:
+        backend = VideoBackend.from_filename(
+            video_path,
+            dataset=backend_metadata.get("dataset", None),
+            grayscale=backend_metadata.get("grayscale", None),
+            input_format=backend_metadata.get("input_format", None),
+        )
+    except ValueError:
+        backend = None
+
+    return Video(
+        filename=video_path, backend=backend, backend_metadata=backend_metadata
+    )
+
+
 def read_videos(labels_path: str) -> list[Video]:
     """Read `Video` dataset in a SLEAP labels file.
 
@@ -49,47 +94,148 @@ def read_videos(labels_path: str) -> list[Video]:
     Returns:
         A list of `Video` objects.
     """
-    # TODO (DS) - Find shape of video
-    videos = [json.loads(x) for x in read_hdf5_dataset(labels_path, "videos_json")]
-    video_objects = []
-    for video in videos:
-        backend = video["backend"]
-        video_path = backend["filename"]
-
-        # Marker for embedded videos.
-        if video_path == ".":
-            video_path = labels_path
-
-        # Basic path resolution.
-        video_path = Path(video_path)
-        if not video_path.exists():
-            # Check for the same filename in the same directory as the labels file.
-            video_path_ = Path(labels_path).parent / video_path.name
-            if video_path_.exists():
-                video_path = video_path_
-            else:
-                # TODO (TP): Expand capabilities of path resolution to support more
-                # complex path finding strategies.
-                pass
-
-        video_path = video_path.as_posix()
-
-        if "filenames" in backend:
-            # This is an ImageVideo.
-            # TODO: Path resolution.
-            video_path = backend["filenames"]
-
-        try:
-            backend = VideoBackend.from_filename(
-                video_path,
-                dataset=backend.get("dataset", None),
-                grayscale=backend.get("grayscale", None),
-                input_format=backend.get("input_format", None),
+    videos = []
+    for video_data in read_hdf5_dataset(labels_path, "videos_json"):
+        video_json = json.loads(video_data)
+        video = make_video(video_json, labels_path)
+        videos.append(video)
+    return videos
+
+
+def video_to_dict(video: Video) -> dict:
+    """Convert a `Video` object to a JSON-compatible dictionary.
+
+    Args:
+        video: A `Video` object to convert.
+
+    Returns:
+        A dictionary containing the video metadata.
+    """
+    if video.backend is None:
+        return {"filename": video.filename, "backend": None}
+
+    if type(video.backend) == MediaVideo:
+        return {
+            "filename": video.filename,
+            "backend": {
+                "type": "MediaVideo",
+                "shape": video.shape,
+                "filename": video.filename,
+                "grayscale": video.grayscale,
+                "bgr": True,
+                "dataset": "",
+                "input_format": "",
+            },
+        }
+
+    elif type(video.backend) == HDF5Video:
+        return {
+            "filename": video.filename,
+            "backend": {
+                "type": "HDF5Video",
+                "shape": video.shape,
+                "filename": (
+                    "." if video.backend.has_embedded_images else video.filename
+                ),
+                "dataset": video.backend.dataset,
+                "input_format": video.backend.input_format,
+                "convert_range": False,
+                "has_embedded_images": video.backend.has_embedded_images,
+            },
+        }
+
+    elif type(video.backend) == ImageVideo:
+        return {
+            "filename": video.filename,
+            "backend": {
+                "type": "ImageVideo",
+                "shape": video.shape,
+                "filename": video.backend.filename[0],
+                "filenames": video.backend.filename,
+                "dataset": getattr(video.backend, "dataset", None),
+                "grayscale": getattr(video.backend, "grayscale", None),
+                "input_format": getattr(video.backend, "input_format", None),
+            },
+        }
+
+
+def embed_video(
+    labels_path: str,
+    video: Video,
+    group: str,
+    frame_inds: list[int],
+    image_format: str = "png",
+    fixed_length: bool = True,
+):
+    """Embed frames of a video in a SLEAP labels file.
+
+    Args:
+        labels_path: A string path to the SLEAP labels file.
+        video: A `Video` object to embed in the labels file.
+        frame_inds: A list of frame indices to embed.
+        group: The name of the group to store the embedded video in. Image data will be
+            stored in a dataset named `{group}/video`. Frame indices will be stored
+            in a data set named `{group}/frame_numbers`.
+        image_format: The image format to use for embedding. Valid formats are "png"
+            (the default), "jpg" or "gzip".
+        fixed_length: If `True` (the default), the embedded images will be padded to the
+            length of the largest image. If `False`, the images will be stored as
+            variable length, which is smaller but may not be supported by all readers.
+    """
+    # Load the image data and optionally encode it.
+    imgs_data = []
+    for frame_idx in frame_inds:
+        frame = video[frame_idx]
+
+        if image_format == "gzip":
+            img_data = frame
+        else:
+            img_data = iio.imwrite(
+                "<bytes>", frame, extension="." + image_format
+            ).astype("int8")
+
+        imgs_data.append(img_data)
+
+    # Write the image data to the labels file.
+    with h5py.File(labels_path, "a") as f:
+        if image_format == "gzip":
+            f.create_dataset(
+                f"{group}/video", data=imgs_data, compression="gzip", chunks=True
             )
-        except ValueError:
-            backend = None
-        video_objects.append(Video(filename=video_path, backend=backend))
-    return video_objects
+        else:
+            if fixed_length:
+                ds = f.create_dataset(
+                    f"{group}/video",
+                    shape=(len(imgs_data), max(len(img) for img in imgs_data)),
+                    dtype="int8",
+                    compression="gzip",
+                )
+                for i, img in enumerate(imgs_data):
+                    ds[i, : len(img)] = img
+            else:
+                ds = f.create_dataset(
+                    f"{group}/video",
+                    shape=(len(imgs_data),),
+                    dtype=h5py.special_dtype(vlen=np.dtype("int8")),
+                )
+                for i, img in enumerate(imgs_data):
+                    ds[i] = img
+
+        # Store metadata.
+        ds.attrs["format"] = image_format
+        (
+            ds.attrs["frames"],
+            ds.attrs["height"],
+            ds.attrs["width"],
+            ds.attrs["channels"],
+        ) = video.shape
+
+        # Store frame indices.
+        f.create_dataset(f"{group}/frame_numbers", data=frame_inds)
+
+        # Store source video.
+        grp = f.require_group(f"{group}/source_video")
+        grp.attrs["json"] = json.dumps(video_to_dict(video), separators=(",", ":"))
 
 
 def write_videos(labels_path: str, videos: list[Video]):
@@ -101,53 +247,13 @@ def write_videos(labels_path: str, videos: list[Video]):
     """
     video_jsons = []
     for video in videos:
-        if type(video.backend) == MediaVideo:
-            video_json = {
-                "backend": {
-                    "filename": video.filename,
-                    "grayscale": video.backend.grayscale,
-                    "bgr": True,
-                    "dataset": "",
-                    "input_format": "",
-                }
-            }
+        video_json = video_to_dict(video)
 
-        elif type(video.backend) == HDF5Video:
-            video_json = {
-                "backend": {
-                    "filename": (
-                        "." if video.backend.has_embedded_images else video.filename
-                    ),
-                    "dataset": video.backend.dataset,
-                    "input_format": video.backend.input_format,
-                    "convert_range": False,
-                }
-            }
+        if type(video.backend) == HDF5Video:
             # TODO: Handle saving embedded images or restoring source video.
             # Ref: https://github.com/talmolab/sleap/blob/fb61b6ce7a9ac9613d99303111f3daafaffc299b/sleap/io/format/hdf5.py#L246-L273
+            pass
 
-        elif type(video.backend) == ImageVideo:
-            shape = video.shape
-            if shape is None:
-                height, width, channels = 0, 0, 1
-            else:
-                height, width, channels = shape[1:]
-
-            video_json = {
-                "backend": {
-                    "filename": video.filename[0],
-                    "filenames": video.filename,
-                    "height_": height,
-                    "width_": width,
-                    "channels_": channels,
-                    "grayscale": video.backend.grayscale,
-                }
-            }
-
-        else:
-            raise NotImplementedError(
-                f"Cannot serialize video backend for video: {video}"
-            )
         video_jsons.append(np.string_(json.dumps(video_json, separators=(",", ":"))))
 
     with h5py.File(labels_path, "a") as f:
diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index 8f2b5b9f..4211fd4a 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -160,7 +160,8 @@ def num_frames(self) -> int:
     @property
     def img_shape(self) -> Tuple[int, int, int]:
         """Shape of a single frame in the video."""
-        return self.get_frame(0).shape
+        height, width, channels = self.get_frame(0).shape
+        return int(height), int(width), int(channels)
 
     @property
     def shape(self) -> Tuple[int, int, int, int]:
@@ -434,20 +435,6 @@ def _read_frames(self, frame_inds: list) -> np.ndarray:
                     )
         return imgs
 
-    def to_json(self) -> dict:
-        """Return JSON-serializable dictionary of the video backend."""
-        return {
-            "type": "MediaVideo",
-            "shape": self.shape,
-            "backend": {
-                "filename": self.filename,
-                "grayscale": self.grayscale,
-                "bgr": True,
-                "dataset": "",
-                "input_format": "",
-            },
-        }
-
 
 @attrs.define
 class HDF5Video(VideoBackend):
@@ -577,7 +564,7 @@ def img_shape(self) -> Tuple[int, int, int]:
                 img_shape = ds.shape[1:]
         if self.input_format == "channels_first":
             img_shape = img_shape[::-1]
-        return img_shape
+        return int(img_shape[0]), int(img_shape[1]), int(img_shape[2])
 
     def read_test_frame(self) -> np.ndarray:
         """Read a single frame from the video to test for grayscale."""
@@ -696,20 +683,6 @@ def _read_frames(self, frame_inds: list) -> np.ndarray:
 
         return imgs
 
-    def to_json(self) -> dict:
-        """Return JSON-serializable dictionary of the video backend."""
-        return {
-            "type": "HDF5Video",
-            "shape": self.shape,
-            "backend": {
-                "filename": ("." if self.has_embedded_images else self.filename),
-                "dataset": self.dataset,
-                "input_format": self.input_format,
-                "convert_range": False,
-                "has_embedded_images": self.has_embedded_images,
-            },
-        }
-
 
 @attrs.define
 class ImageVideo(VideoBackend):
@@ -754,25 +727,3 @@ def _read_frame(self, frame_idx: int) -> np.ndarray:
         if img.ndim == 2:
             img = np.expand_dims(img, axis=-1)
         return img
-
-    def to_json(self) -> dict:
-        """Return JSON-serializable dictionary of the video backend."""
-
-        shape = self.shape
-        if shape is None:
-            height, width, channels = 0, 0, 1
-        else:
-            height, width, channels = shape[1:]
-
-        return {
-            "type": "ImageVideo",
-            "shape": self.shape,
-            "backend": {
-                "filename": self.filename[0],
-                "filenames": self.filename,
-                "height_": height,
-                "width_": width,
-                "channels_": channels,
-                "grayscale": self.grayscale,
-            },
-        }
diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index 46d81473..d0f2d284 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -26,12 +26,16 @@ class Video:
         filename: The filename(s) of the video.
         backend: An object that implements the basic methods for reading and
             manipulating frames of a specific video type.
+        backend_metadata: A dictionary of metadata specific to the backend. This is
+            useful for storing metadata that requires an open backend (e.g., shape
+            information) without having access to the video file itself.
 
     See also: VideoBackend
     """
 
     filename: str | list[str]
     backend: Optional[VideoBackend] = None
+    backend_metadata: dict[str, any] = {}
 
     EXTS = MediaVideo.EXTS + HDF5Video.EXTS
 
@@ -88,6 +92,23 @@ def _get_shape(self) -> Tuple[int, int, int, int] | None:
         try:
             return self.backend.shape
         except:
+            if "shape" in self.backend_metadata:
+                return self.backend_metadata["shape"]
+            return None
+
+    @property
+    def grayscale(self) -> bool | None:
+        """Return whether the video is grayscale.
+
+        If the video backend is not set or it cannot determine whether the video is
+        grayscale, this will return None.
+        """
+        shape = self.shape
+        if shape is not None:
+            return shape[-1] == 1
+        else:
+            if "grayscale" in self.backend_metadata:
+                return self.backend_metadata["grayscale"]
             return None
 
     def __len__(self) -> int:
@@ -189,6 +210,12 @@ def open(
             if grayscale is None:
                 grayscale = getattr(self.backend, "grayscale", None)
 
+        else:
+            if "dataset" in self.backend_metadata:
+                dataset = self.backend_metadata["dataset"]
+            if "grayscale" in self.backend_metadata:
+                grayscale = self.backend_metadata["grayscale"]
+
         # Close previous backend if open.
         self.close()
 
@@ -231,10 +258,3 @@ def replace_filename(
                 self.open()
             else:
                 self.close()
-
-    def to_json(self) -> dict:
-        """Return a dictionary representation of the video."""
-        if self.backend is not None:
-            return self.backend.to_json()
-        else:
-            return {"filename": self.filename, "backend": None}
diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index e2ab9470..8a90694e 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -32,6 +32,8 @@
 )
 from sleap_io.io.utils import read_hdf5_dataset
 import numpy as np
+import simplejson as json
+
 
 from sleap_io.io.video import ImageVideo
 
@@ -98,17 +100,26 @@ def test_read_videos_pkg(slp_minimal_pkg):
 
 
 def test_write_videos(slp_minimal_pkg, centered_pair, tmp_path):
+
+    def load_jsons(h5_path, dataset):
+        return [json.loads(x) for x in read_hdf5_dataset(h5_path, dataset)]
+
+    def compare_jsons(jsons_ref, jsons_test):
+        for jsons_ref, jsons_test in zip(jsons_ref, jsons_test):
+            for k in jsons_ref["backend"]:
+                assert jsons_ref["backend"][k] == jsons_test["backend"][k]
+
     videos = read_videos(slp_minimal_pkg)
     write_videos(tmp_path / "test_minimal_pkg.slp", videos)
-    json_fixture = read_hdf5_dataset(slp_minimal_pkg, "videos_json")
-    json_test = read_hdf5_dataset(tmp_path / "test_minimal_pkg.slp", "videos_json")
-    assert json_fixture == json_test
+    json_fixture = load_jsons(slp_minimal_pkg, "videos_json")
+    json_test = load_jsons(tmp_path / "test_minimal_pkg.slp", "videos_json")
+    compare_jsons(json_fixture, json_test)
 
     videos = read_videos(centered_pair)
     write_videos(tmp_path / "test_centered_pair.slp", videos)
-    json_fixture = read_hdf5_dataset(centered_pair, "videos_json")
-    json_test = read_hdf5_dataset(tmp_path / "test_centered_pair.slp", "videos_json")
-    assert json_fixture == json_test
+    json_fixture = load_jsons(centered_pair, "videos_json")
+    json_test = load_jsons(tmp_path / "test_centered_pair.slp", "videos_json")
+    compare_jsons(json_fixture, json_test)
 
     videos = read_videos(centered_pair) * 2
     write_videos(tmp_path / "test_centered_pair_2vids.slp", videos)

From 84f954279950847c2bb39ff1c628ef1a0166d92b Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 21:42:30 -0700
Subject: [PATCH 03/29] Try to restore source video when available

---
 sleap_io/io/slp.py      | 37 +++++++++++++++++++++++++++++++------
 sleap_io/model/video.py | 12 ++++++++++--
 2 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index cd5cbd12..4ece91b9 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -37,19 +37,23 @@ class InstanceType(IntEnum):
     PREDICTED = 1
 
 
-def make_video(video_json: dict, labels_path: str) -> Video:
+def make_video(labels_path: str, video_json: dict, video_index: int) -> Video:
     """Create a `Video` object from a JSON dictionary.
 
     Args:
-        video_json: A dictionary containing the video metadata.
         labels_path: A string path to the SLEAP labels file.
+        video_json: A dictionary containing the video metadata.
+        video_index: The index of the video in the labels file.
     """
     backend_metadata = video_json["backend"]
     video_path = backend_metadata["filename"]
 
     # Marker for embedded videos.
+    source_video = None
+    is_embedded = False
     if video_path == ".":
         video_path = labels_path
+        is_embedded = True
 
     # Basic path resolution.
     video_path = Path(video_path)
@@ -63,6 +67,16 @@ def make_video(video_json: dict, labels_path: str) -> Video:
             # complex path finding strategies.
             pass
 
+    if is_embedded and video_path.exists():
+        # Try to recover the source video.
+        with h5py.File(labels_path, "r") as f:
+            if f"video{video_index}" in f:
+                source_video_json = json.loads(
+                    f[f"video{video_index}/source_video"].attrs["json"]
+                )
+                source_video = make_video(labels_path, source_video_json, video_index)
+
+    # Convert video path to string.
     video_path = video_path.as_posix()
 
     if "filenames" in backend_metadata:
@@ -81,7 +95,10 @@ def make_video(video_json: dict, labels_path: str) -> Video:
         backend = None
 
     return Video(
-        filename=video_path, backend=backend, backend_metadata=backend_metadata
+        filename=video_path,
+        backend=backend,
+        backend_metadata=backend_metadata,
+        source_video=source_video,
     )
 
 
@@ -95,9 +112,11 @@ def read_videos(labels_path: str) -> list[Video]:
         A list of `Video` objects.
     """
     videos = []
-    for video_data in read_hdf5_dataset(labels_path, "videos_json"):
+    for video_ind, video_data in enumerate(
+        read_hdf5_dataset(labels_path, "videos_json")
+    ):
         video_json = json.loads(video_data)
-        video = make_video(video_json, labels_path)
+        video = make_video(labels_path, video_json, video_index=video_ind)
         videos.append(video)
     return videos
 
@@ -234,8 +253,14 @@ def embed_video(
         f.create_dataset(f"{group}/frame_numbers", data=frame_inds)
 
         # Store source video.
+        if video.source_video is not None:
+            source_video = video.source_video
+        else:
+            source_video = video
         grp = f.require_group(f"{group}/source_video")
-        grp.attrs["json"] = json.dumps(video_to_dict(video), separators=(",", ":"))
+        grp.attrs["json"] = json.dumps(
+            video_to_dict(source_video), separators=(",", ":")
+        )
 
 
 def write_videos(labels_path: str, videos: list[Video]):
diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index d0f2d284..3bd46913 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -29,6 +29,8 @@ class Video:
         backend_metadata: A dictionary of metadata specific to the backend. This is
             useful for storing metadata that requires an open backend (e.g., shape
             information) without having access to the video file itself.
+        source_video: The source video object if this is a proxy video. This is present
+            when the video contains an embedded subset of frames from another video.
 
     See also: VideoBackend
     """
@@ -36,6 +38,7 @@ class Video:
     filename: str | list[str]
     backend: Optional[VideoBackend] = None
     backend_metadata: dict[str, any] = {}
+    source_video: Optional[Video] = None
 
     EXTS = MediaVideo.EXTS + HDF5Video.EXTS
 
@@ -46,6 +49,7 @@ def from_filename(
         dataset: Optional[str] = None,
         grayscale: Optional[bool] = None,
         keep_open: bool = True,
+        source_video: Optional[Video] = None,
         **kwargs,
     ) -> VideoBackend:
         """Create a Video from a filename.
@@ -59,6 +63,9 @@ def from_filename(
                 frames. If False, will close the reader after each call. If True (the
                 default), it will keep the reader open and cache it for subsequent calls
                 which may enhance the performance of reading multiple frames.
+            source_video: The source video object if this is a proxy video. This is
+                present when the video contains an embedded subset of frames from
+                another video.
 
         Returns:
             Video instance with the appropriate backend instantiated.
@@ -72,6 +79,7 @@ def from_filename(
                 keep_open=keep_open,
                 **kwargs,
             ),
+            source_video=source_video,
         )
 
     @property
@@ -211,9 +219,9 @@ def open(
                 grayscale = getattr(self.backend, "grayscale", None)
 
         else:
-            if "dataset" in self.backend_metadata:
+            if dataset is None and "dataset" in self.backend_metadata:
                 dataset = self.backend_metadata["dataset"]
-            if "grayscale" in self.backend_metadata:
+            if grayscale is None and "grayscale" in self.backend_metadata:
                 grayscale = self.backend_metadata["grayscale"]
 
         # Close previous backend if open.

From a15f44bfe90d60fc9dd1c73885b042d1711093d0 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 21:45:53 -0700
Subject: [PATCH 04/29] Rename symbol

---
 sleap_io/io/slp.py | 25 +++++++++++++++----------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 4ece91b9..ecb81d91 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -37,13 +37,16 @@ class InstanceType(IntEnum):
     PREDICTED = 1
 
 
-def make_video(labels_path: str, video_json: dict, video_index: int) -> Video:
+def make_video(
+    labels_path: str, video_json: dict, video_ind: int | None = None
+) -> Video:
     """Create a `Video` object from a JSON dictionary.
 
     Args:
         labels_path: A string path to the SLEAP labels file.
         video_json: A dictionary containing the video metadata.
-        video_index: The index of the video in the labels file.
+        video_ind: The index of the video in the labels file. This is used to try to
+            recover the source video for embedded videos. This is skipped if `None`.
     """
     backend_metadata = video_json["backend"]
     video_path = backend_metadata["filename"]
@@ -67,17 +70,19 @@ def make_video(labels_path: str, video_json: dict, video_index: int) -> Video:
             # complex path finding strategies.
             pass
 
-    if is_embedded and video_path.exists():
+    # Convert video path to string.
+    video_path = video_path.as_posix()
+
+    if is_embedded:
         # Try to recover the source video.
         with h5py.File(labels_path, "r") as f:
-            if f"video{video_index}" in f:
+            if f"video{video_ind}" in f:
                 source_video_json = json.loads(
-                    f[f"video{video_index}/source_video"].attrs["json"]
+                    f[f"video{video_ind}/source_video"].attrs["json"]
+                )
+                source_video = make_video(
+                    labels_path, source_video_json, video_ind=None
                 )
-                source_video = make_video(labels_path, source_video_json, video_index)
-
-    # Convert video path to string.
-    video_path = video_path.as_posix()
 
     if "filenames" in backend_metadata:
         # This is an ImageVideo.
@@ -116,7 +121,7 @@ def read_videos(labels_path: str) -> list[Video]:
         read_hdf5_dataset(labels_path, "videos_json")
     ):
         video_json = json.loads(video_data)
-        video = make_video(labels_path, video_json, video_index=video_ind)
+        video = make_video(labels_path, video_json, video_ind=video_ind)
         videos.append(video)
     return videos
 

From 2dbaea023b745b237e5193700b6777b4f46a8f47 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 21:54:29 -0700
Subject: [PATCH 05/29] Use backend metadata when available when serializing

---
 sleap_io/io/slp.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index ecb81d91..0b7bb630 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -136,7 +136,7 @@ def video_to_dict(video: Video) -> dict:
         A dictionary containing the video metadata.
     """
     if video.backend is None:
-        return {"filename": video.filename, "backend": None}
+        return {"filename": video.filename, "backend": video.backend_metadata}
 
     if type(video.backend) == MediaVideo:
         return {
@@ -176,9 +176,9 @@ def video_to_dict(video: Video) -> dict:
                 "shape": video.shape,
                 "filename": video.backend.filename[0],
                 "filenames": video.backend.filename,
-                "dataset": getattr(video.backend, "dataset", None),
-                "grayscale": getattr(video.backend, "grayscale", None),
-                "input_format": getattr(video.backend, "input_format", None),
+                "dataset": video.backend_metadata.get("dataset", None),
+                "grayscale": video.grayscale,
+                "input_format": video.backend_metadata.get("input_format", None),
             },
         }
 

From d933cc0e55dee9352f6a99300781db30505d9ce3 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 21:54:54 -0700
Subject: [PATCH 06/29] Fix backend metadata factory

---
 sleap_io/model/video.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index 3bd46913..48236d0b 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -5,14 +5,14 @@
 """
 
 from __future__ import annotations
-from attrs import define
+import attrs
 from typing import Tuple, Optional, Optional
 import numpy as np
 from sleap_io.io.video import VideoBackend, MediaVideo, HDF5Video
 from pathlib import Path
 
 
-@define
+@attrs.define
 class Video:
     """`Video` class used by sleap to represent videos and data associated with them.
 
@@ -37,7 +37,7 @@ class Video:
 
     filename: str | list[str]
     backend: Optional[VideoBackend] = None
-    backend_metadata: dict[str, any] = {}
+    backend_metadata: dict[str, any] = attrs.field(factory=dict)
     source_video: Optional[Video] = None
 
     EXTS = MediaVideo.EXTS + HDF5Video.EXTS

From 247f92e0de13a475148f93d5c1da574f6edb6a52 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 22:17:56 -0700
Subject: [PATCH 07/29] Re-embed videos when saving labels with embedded videos

---
 sleap_io/io/slp.py   | 22 ++++++++++++++--------
 sleap_io/io/video.py | 14 +++++++++-----
 2 files changed, 23 insertions(+), 13 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 0b7bb630..9d11d3ab 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -201,7 +201,7 @@ def embed_video(
             stored in a dataset named `{group}/video`. Frame indices will be stored
             in a data set named `{group}/frame_numbers`.
         image_format: The image format to use for embedding. Valid formats are "png"
-            (the default), "jpg" or "gzip".
+            (the default), "jpg" or "hdf5".
         fixed_length: If `True` (the default), the embedded images will be padded to the
             length of the largest image. If `False`, the images will be stored as
             variable length, which is smaller but may not be supported by all readers.
@@ -211,7 +211,7 @@ def embed_video(
     for frame_idx in frame_inds:
         frame = video[frame_idx]
 
-        if image_format == "gzip":
+        if image_format == "hdf5":
             img_data = frame
         else:
             img_data = iio.imwrite(
@@ -222,7 +222,7 @@ def embed_video(
 
     # Write the image data to the labels file.
     with h5py.File(labels_path, "a") as f:
-        if image_format == "gzip":
+        if image_format == "hdf5":
             f.create_dataset(
                 f"{group}/video", data=imgs_data, compression="gzip", chunks=True
             )
@@ -259,6 +259,7 @@ def embed_video(
 
         # Store source video.
         if video.source_video is not None:
+            # If this is already an embedded dataset, retain the previous source video.
             source_video = video.source_video
         else:
             source_video = video
@@ -276,13 +277,18 @@ def write_videos(labels_path: str, videos: list[Video]):
         videos: A list of `Video` objects to store the metadata for.
     """
     video_jsons = []
-    for video in videos:
+    for video_ind, video in enumerate(videos):
         video_json = video_to_dict(video)
 
-        if type(video.backend) == HDF5Video:
-            # TODO: Handle saving embedded images or restoring source video.
-            # Ref: https://github.com/talmolab/sleap/blob/fb61b6ce7a9ac9613d99303111f3daafaffc299b/sleap/io/format/hdf5.py#L246-L273
-            pass
+        if type(video.backend) == HDF5Video and video.backend.has_embedded_images:
+            # If the video is an HDF5Video with embedded images, embed the images again.
+            embed_video(
+                labels_path,
+                video,
+                group=f"video{video_ind}",
+                frame_inds=video.backend.source_inds,
+                image_format=video.backend.image_format,
+            )
 
         video_jsons.append(np.string_(json.dumps(video_json, separators=(",", ":"))))
 
diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index 4211fd4a..9323a696 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -479,6 +479,8 @@ class HDF5Video(VideoBackend):
             when reading embedded image datasets.
         source_inds: Indices of the frames in the source video file. This is metadata
             and only used when reading embedded image datasets.
+        image_format: Format of the images in the embedded dataset. This is metadata and
+            only used when reading embedded image datasets.
     """
 
     dataset: Optional[str] = None
@@ -489,6 +491,7 @@ class HDF5Video(VideoBackend):
     frame_map: dict[int, int] = attrs.field(init=False, default=attrs.Factory(dict))
     source_filename: Optional[str] = None
     source_inds: Optional[np.ndarray] = None
+    image_format: str = "hdf5"
 
     EXTS = ("h5", "hdf5", "slp")
 
@@ -531,6 +534,9 @@ def find_embedded(name, obj):
             # This may be an embedded video dataset. Check for frame map.
             ds = f[self.dataset]
 
+            if "format" in ds.attrs:
+                self.image_format = ds.attrs["format"]
+
             if "frame_numbers" in ds.parent:
                 frame_numbers = ds.parent["frame_numbers"][:]
                 self.frame_map = {frame: idx for idx, frame in enumerate(frame_numbers)}
@@ -577,9 +583,7 @@ def read_test_frame(self) -> np.ndarray:
     @property
     def has_embedded_images(self) -> bool:
         """Return True if the dataset contains embedded images."""
-        with h5py.File(self.filename, "r") as f:
-            ds = f[self.dataset]
-            return "format" in ds.attrs
+        return self.image_format is not None and self.image_format != "hdf5"
 
     def decode_embedded(self, img_string: np.ndarray, format: str) -> np.ndarray:
         """Decode an embedded image string into a numpy array.
@@ -633,8 +637,8 @@ def _read_frame(self, frame_idx: int) -> np.ndarray:
 
         img = ds[frame_idx]
 
-        if "format" in ds.attrs:
-            img = self.decode_embedded(img, ds.attrs["format"])
+        if self.has_embedded_images:
+            img = self.decode_embedded(img)
 
         if self.input_format == "channels_first":
             img = np.transpose(img, (2, 1, 0))

From d35f07e9e04d833b9ad8f9d3fa4b4246110658a7 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 22:41:28 -0700
Subject: [PATCH 08/29] Fix serialization and logic for checking for embedded
 images

---
 sleap_io/io/slp.py   | 22 ++++++++++++++++++----
 sleap_io/io/video.py |  5 ++---
 tests/io/test_slp.py | 40 +++++++++++++++++-----------------------
 3 files changed, 37 insertions(+), 30 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 9d11d3ab..0ac3d252 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -25,9 +25,15 @@
     read_hdf5_attrs,
     read_hdf5_dataset,
 )
-import imageio.v3 as iio
 from enum import IntEnum
 from pathlib import Path
+import imageio.v3 as iio
+import sys
+
+try:
+    import cv2
+except ImportError:
+    pass
 
 
 class InstanceType(IntEnum):
@@ -214,9 +220,17 @@ def embed_video(
         if image_format == "hdf5":
             img_data = frame
         else:
-            img_data = iio.imwrite(
-                "<bytes>", frame, extension="." + image_format
-            ).astype("int8")
+            if "cv2" in sys.modules:
+                img_data = np.squeeze(
+                    cv2.imencode("." + image_format, frame)[1]
+                ).astype("int8")
+            else:
+                img_data = np.frombuffer(
+                    iio.imwrite(
+                        "<bytes>", frame.squeeze(axis=-1), extension="." + image_format
+                    ),
+                    dtype="int8",
+                )
 
         imgs_data.append(img_data)
 
diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index 9323a696..fae09234 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -585,13 +585,12 @@ def has_embedded_images(self) -> bool:
         """Return True if the dataset contains embedded images."""
         return self.image_format is not None and self.image_format != "hdf5"
 
-    def decode_embedded(self, img_string: np.ndarray, format: str) -> np.ndarray:
+    def decode_embedded(self, img_string: np.ndarray) -> np.ndarray:
         """Decode an embedded image string into a numpy array.
 
         Args:
             img_string: Binary string of the image as a `int8` numpy vector with the
                 bytes as values corresponding to the format-encoded image.
-            format: Image format (e.g., "png" or "jpg").
 
         Returns:
             The decoded image as a numpy array of shape `(height, width, channels)`. If
@@ -604,7 +603,7 @@ def decode_embedded(self, img_string: np.ndarray, format: str) -> np.ndarray:
         if "cv2" in sys.modules:
             img = cv2.imdecode(img_string, cv2.IMREAD_UNCHANGED)
         else:
-            img = iio.imread(BytesIO(img_string), extension=f".{format}")
+            img = iio.imread(BytesIO(img_string), extension=f".{self.image_format}")
 
         if img.ndim == 2:
             img = np.expand_dims(img, axis=-1)
diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index 8a90694e..766692d6 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -101,32 +101,26 @@ def test_read_videos_pkg(slp_minimal_pkg):
 
 def test_write_videos(slp_minimal_pkg, centered_pair, tmp_path):
 
-    def load_jsons(h5_path, dataset):
-        return [json.loads(x) for x in read_hdf5_dataset(h5_path, dataset)]
-
-    def compare_jsons(jsons_ref, jsons_test):
-        for jsons_ref, jsons_test in zip(jsons_ref, jsons_test):
-            for k in jsons_ref["backend"]:
-                assert jsons_ref["backend"][k] == jsons_test["backend"][k]
-
-    videos = read_videos(slp_minimal_pkg)
-    write_videos(tmp_path / "test_minimal_pkg.slp", videos)
-    json_fixture = load_jsons(slp_minimal_pkg, "videos_json")
-    json_test = load_jsons(tmp_path / "test_minimal_pkg.slp", "videos_json")
-    compare_jsons(json_fixture, json_test)
-
-    videos = read_videos(centered_pair)
-    write_videos(tmp_path / "test_centered_pair.slp", videos)
-    json_fixture = load_jsons(centered_pair, "videos_json")
-    json_test = load_jsons(tmp_path / "test_centered_pair.slp", "videos_json")
-    compare_jsons(json_fixture, json_test)
+    def compare_videos(videos_ref, videos_test):
+        assert len(videos_ref) == len(videos_test)
+        for video_ref, video_test in zip(videos_ref, videos_test):
+            assert video_ref.shape == video_test.shape
+            assert (video_ref[0] == video_test[0]).all()
+
+    videos_ref = read_videos(slp_minimal_pkg)
+    write_videos(tmp_path / "test_minimal_pkg.slp", videos_ref)
+    videos_test = read_videos(tmp_path / "test_minimal_pkg.slp")
+    compare_videos(videos_ref, videos_test)
+
+    videos_ref = read_videos(centered_pair)
+    write_videos(tmp_path / "test_centered_pair.slp", videos_ref)
+    videos_test = read_videos(tmp_path / "test_centered_pair.slp")
+    compare_videos(videos_ref, videos_test)
 
     videos = read_videos(centered_pair) * 2
     write_videos(tmp_path / "test_centered_pair_2vids.slp", videos)
-    json_test = read_hdf5_dataset(
-        tmp_path / "test_centered_pair_2vids.slp", "videos_json"
-    )
-    assert len(json_test) == 2
+    videos_test = read_videos(tmp_path / "test_centered_pair_2vids.slp")
+    compare_videos(videos, videos_test)
 
 
 def test_write_tracks(centered_pair, tmp_path):

From 31341da0993339bdd81df431d31f938053f5665a Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 22:45:59 -0700
Subject: [PATCH 09/29] Fix multi-frame decoding

---
 sleap_io/io/video.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index fae09234..b295849e 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -674,7 +674,7 @@ def _read_frames(self, frame_inds: list) -> np.ndarray:
 
         if "format" in ds.attrs:
             imgs = np.stack(
-                [self.decode_embedded(img, ds.attrs["format"]) for img in imgs],
+                [self.decode_embedded(img) for img in imgs],
                 axis=0,
             )
 

From 4402cb43cd3dae86f02b88b85ff0aef100882e74 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 23:13:38 -0700
Subject: [PATCH 10/29] Fix docstring order

---
 sleap_io/io/slp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 0ac3d252..92293833 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -202,10 +202,10 @@ def embed_video(
     Args:
         labels_path: A string path to the SLEAP labels file.
         video: A `Video` object to embed in the labels file.
-        frame_inds: A list of frame indices to embed.
         group: The name of the group to store the embedded video in. Image data will be
             stored in a dataset named `{group}/video`. Frame indices will be stored
             in a data set named `{group}/frame_numbers`.
+        frame_inds: A list of frame indices to embed.
         image_format: The image format to use for embedding. Valid formats are "png"
             (the default), "jpg" or "hdf5".
         fixed_length: If `True` (the default), the embedded images will be padded to the

From 696ae60969ab31e7d3dfdc855651f87567bccc54 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 23:46:59 -0700
Subject: [PATCH 11/29] Add method to embed a list of frames and update the
 objects

---
 sleap_io/io/slp.py | 92 +++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 82 insertions(+), 10 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 92293833..26b193a4 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -196,7 +196,7 @@ def embed_video(
     frame_inds: list[int],
     image_format: str = "png",
     fixed_length: bool = True,
-):
+) -> Video:
     """Embed frames of a video in a SLEAP labels file.
 
     Args:
@@ -211,6 +211,12 @@ def embed_video(
         fixed_length: If `True` (the default), the embedded images will be padded to the
             length of the largest image. If `False`, the images will be stored as
             variable length, which is smaller but may not be supported by all readers.
+
+    Returns:
+        An embedded `Video` object.
+
+        If the video is already embedded, the original video will be returned. If not,
+        a new `Video` object will be created with the embedded data.
     """
     # Load the image data and optionally encode it.
     imgs_data = []
@@ -275,13 +281,73 @@ def embed_video(
         if video.source_video is not None:
             # If this is already an embedded dataset, retain the previous source video.
             source_video = video.source_video
+            embedded_video = video
         else:
             source_video = video
+            embedded_video = Video(
+                filename=labels_path,
+                backend=VideoBackend.from_filename(
+                    labels_path,
+                    dataset=f"{group}/video",
+                    grayscale=video.grayscale,
+                    keep_open=False,
+                ),
+                source_video=source_video,
+            )
+
         grp = f.require_group(f"{group}/source_video")
         grp.attrs["json"] = json.dumps(
             video_to_dict(source_video), separators=(",", ":")
         )
 
+        return embedded_video
+
+
+def embed_frames(
+    labels_path: str,
+    labels: Labels,
+    to_embed: list[tuple[Video, int]],
+    image_format: str = "png",
+):
+    """Embed frames in a SLEAP labels file.
+
+    Args:
+        labels_path: A string path to the SLEAP labels file.
+        labels: A `Labels` object to embed in the labels file.
+        to_embed: A list of tuples of `(video, frame_idx)` specifying the frames to
+            embed.
+        image_format: The image format to use for embedding. Valid formats are "png"
+            (the default), "jpg" or "hdf5".
+
+    Notes:
+        This function will embed the frames in the labels file and update the `Videos`
+        and `Labels` objects in place.
+    """
+    to_embed_by_video = {}
+    for video, frame_idx in to_embed:
+        if video not in to_embed_by_video:
+            to_embed_by_video[video] = []
+        to_embed_by_video[video].append(frame_idx)
+
+    replaced_videos = {}
+    for video, frame_inds in to_embed_by_video.items():
+        video_ind = labels.videos.index(video)
+        embedded_video = embed_video(
+            labels_path,
+            video,
+            group=f"video{video_ind}",
+            frame_inds=frame_inds,
+            image_format=image_format,
+        )
+
+        labels.videos[video_ind] = embedded_video
+        replaced_videos[video] = embedded_video
+
+    # Update the labeled frames with the new videos.
+    for lf in labels:
+        if lf.video in replaced_videos:
+            lf.video = replaced_videos[lf.video]
+
 
 def write_videos(labels_path: str, videos: list[Video]):
     """Write video metadata to a SLEAP labels file.
@@ -292,17 +358,23 @@ def write_videos(labels_path: str, videos: list[Video]):
     """
     video_jsons = []
     for video_ind, video in enumerate(videos):
-        video_json = video_to_dict(video)
 
         if type(video.backend) == HDF5Video and video.backend.has_embedded_images:
-            # If the video is an HDF5Video with embedded images, embed the images again.
-            embed_video(
-                labels_path,
-                video,
-                group=f"video{video_ind}",
-                frame_inds=video.backend.source_inds,
-                image_format=video.backend.image_format,
-            )
+            # If the video has embedded images, embed them images again if we haven't
+            # already.
+            with h5py.File(labels_path, "r") as f:
+                already_embedded = f"video{video_ind}/video" in f
+
+            if not already_embedded:
+                video = embed_video(
+                    labels_path,
+                    video,
+                    group=f"video{video_ind}",
+                    frame_inds=video.backend.source_inds,
+                    image_format=video.backend.image_format,
+                )
+
+        video_json = video_to_dict(video)
 
         video_jsons.append(np.string_(json.dumps(video_json, separators=(",", ":"))))
 

From fd9bae9e0e498148a048eaaf7140caec0a6f7893 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sat, 4 May 2024 23:54:51 -0700
Subject: [PATCH 12/29] Fix order of operations

---
 sleap_io/io/slp.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 26b193a4..fe795809 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -300,7 +300,7 @@ def embed_video(
             video_to_dict(source_video), separators=(",", ":")
         )
 
-        return embedded_video
+    return embedded_video
 
 
 def embed_frames(
@@ -362,8 +362,10 @@ def write_videos(labels_path: str, videos: list[Video]):
         if type(video.backend) == HDF5Video and video.backend.has_embedded_images:
             # If the video has embedded images, embed them images again if we haven't
             # already.
-            with h5py.File(labels_path, "r") as f:
-                already_embedded = f"video{video_ind}/video" in f
+            already_embedded = False
+            if Path(labels_path).exists():
+                with h5py.File(labels_path, "r") as f:
+                    already_embedded = f"video{video_ind}/video" in f
 
             if not already_embedded:
                 video = embed_video(

From b41aa38076450e838e2bf64e207d4ebc63896e84 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Sun, 5 May 2024 23:21:17 -0700
Subject: [PATCH 13/29] Add embed_videos

---
 sleap_io/io/slp.py | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index fe795809..e0074c2c 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -349,6 +349,33 @@ def embed_frames(
             lf.video = replaced_videos[lf.video]
 
 
+def embed_videos(
+    labels_path: str, labels: Labels, to_embed: str | list[tuple[Video, int]]
+):
+    """Embed videos in a SLEAP labels file.
+
+    Args:
+        labels_path: A string path to the SLEAP labels file to save.
+        labels: A `Labels` object to save.
+        to_embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, or list of
+            tuples of `(video, frame_idx)` specifying the frames to embed.
+    """
+
+    if to_embed == "user":
+        to_embed = [(lf.video, lf.frame_idx) for lf in labels]
+    elif to_embed == "suggestions":
+        to_embed = [(sf.video, sf.frame_idx) for sf in labels.suggestions]
+    elif to_embed == "user+suggestions":
+        to_embed = [(lf.video, lf.frame_idx) for lf in labels]
+        to_embed += [(sf.video, sf.frame_idx) for sf in labels.suggestions]
+    elif isinstance(to_embed, list):
+        to_embed = to_embed
+    else:
+        raise ValueError(f"Invalid value for to_embed: {to_embed}")
+
+    embed_frames(labels_path, labels, to_embed)
+
+
 def write_videos(labels_path: str, videos: list[Video]):
     """Write video metadata to a SLEAP labels file.
 
@@ -981,15 +1008,22 @@ def read_labels(labels_path: str) -> Labels:
     return labels
 
 
-def write_labels(labels_path: str, labels: Labels):
+def write_labels(
+    labels_path: str, labels: Labels, embed: str | list[tuple[Video, int]] | None = None
+):
     """Write a SLEAP labels file.
 
     Args:
         labels_path: A string path to the SLEAP labels file to save.
         labels: A `Labels` object to save.
+        embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, or list of tuples
+            of `(video, frame_idx)` specifying the frames to embed. If `None`, no frames
+            will be embedded. Existing embedded videos will be re-saved regardless.
     """
     if Path(labels_path).exists():
         Path(labels_path).unlink()
+    if embed is not None:
+        embed_videos(labels_path, labels, embed)
     write_videos(labels_path, labels.videos)
     write_tracks(labels_path, labels.tracks)
     write_suggestions(labels_path, labels.suggestions, labels.videos)

From c8c5ebbcfde728623fdd9c654db3f7b9ec25b091 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:36:55 -0700
Subject: [PATCH 14/29] Fix mid-level embedding function

---
 sleap_io/io/slp.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index e0074c2c..3ce4258d 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -282,6 +282,7 @@ def embed_video(
             # If this is already an embedded dataset, retain the previous source video.
             source_video = video.source_video
             embedded_video = video
+            video.replace_filename(labels_path, open=False)
         else:
             source_video = video
             embedded_video = Video(
@@ -348,6 +349,11 @@ def embed_frames(
         if lf.video in replaced_videos:
             lf.video = replaced_videos[lf.video]
 
+    # Update suggestions with the new videos.
+    for sf in labels.suggestions:
+        if sf.video in replaced_videos:
+            sf.video = replaced_videos[sf.video]
+
 
 def embed_videos(
     labels_path: str, labels: Labels, to_embed: str | list[tuple[Video, int]]
@@ -362,11 +368,11 @@ def embed_videos(
     """
 
     if to_embed == "user":
-        to_embed = [(lf.video, lf.frame_idx) for lf in labels]
+        to_embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
     elif to_embed == "suggestions":
         to_embed = [(sf.video, sf.frame_idx) for sf in labels.suggestions]
     elif to_embed == "user+suggestions":
-        to_embed = [(lf.video, lf.frame_idx) for lf in labels]
+        to_embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
         to_embed += [(sf.video, sf.frame_idx) for sf in labels.suggestions]
     elif isinstance(to_embed, list):
         to_embed = to_embed

From 5f70e1cceba4903e4c12bfe955f590b5178b8026 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:37:15 -0700
Subject: [PATCH 15/29] Hash videos by ID

---
 sleap_io/model/video.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index 48236d0b..04332e28 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -12,7 +12,7 @@
 from pathlib import Path
 
 
-@attrs.define
+@attrs.define(eq=False)
 class Video:
     """`Video` class used by sleap to represent videos and data associated with them.
 
@@ -32,6 +32,11 @@ class Video:
         source_video: The source video object if this is a proxy video. This is present
             when the video contains an embedded subset of frames from another video.
 
+    Notes:
+        Instances of this class are hashed by identity, not by value. This means that
+        two `Video` instances with the same attributes will NOT be considered equal in a
+        set or dict.
+
     See also: VideoBackend
     """
 

From 8fe51c0f3d8b348994efe8c5ec72e29adcda99d5 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:37:47 -0700
Subject: [PATCH 16/29] Add property to return embedded frame indices

---
 sleap_io/io/video.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sleap_io/io/video.py b/sleap_io/io/video.py
index b295849e..74c3d316 100644
--- a/sleap_io/io/video.py
+++ b/sleap_io/io/video.py
@@ -585,6 +585,11 @@ def has_embedded_images(self) -> bool:
         """Return True if the dataset contains embedded images."""
         return self.image_format is not None and self.image_format != "hdf5"
 
+    @property
+    def embedded_frame_inds(self) -> list[int]:
+        """Return the frame indices of the embedded images."""
+        return list(self.frame_map.keys())
+
     def decode_embedded(self, img_string: np.ndarray) -> np.ndarray:
         """Decode an embedded image string into a numpy array.
 

From c329badfd8edec566aa7fd50ec501f3b5f6971ee Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:38:16 -0700
Subject: [PATCH 17/29] Hash LabeledFrame by ID and add convenience checks for
 instance types

---
 sleap_io/model/labeled_frame.py | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/sleap_io/model/labeled_frame.py b/sleap_io/model/labeled_frame.py
index e3b131e0..959f3360 100644
--- a/sleap_io/model/labeled_frame.py
+++ b/sleap_io/model/labeled_frame.py
@@ -11,14 +11,19 @@
 import numpy as np
 
 
-@define(auto_attribs=True)
+@define(eq=False)
 class LabeledFrame:
     """Labeled data for a single frame of a video.
 
     Attributes:
-        video: The :class:`Video` associated with this `LabeledFrame`.
+        video: The `Video` associated with this `LabeledFrame`.
         frame_idx: The index of the `LabeledFrame` in the `Video`.
         instances: List of `Instance` objects associated with this `LabeledFrame`.
+
+    Notes:
+        Instances of this class are hashed by identity, not by value. This means that
+        two `LabeledFrame` instances with the same attributes will NOT be considered
+        equal in a set or dict.
     """
 
     video: Video
@@ -42,11 +47,27 @@ def user_instances(self) -> list[Instance]:
         """Frame instances that are user-labeled (`Instance` objects)."""
         return [inst for inst in self.instances if type(inst) == Instance]
 
+    @property
+    def has_user_instances(self) -> bool:
+        """Return True if the frame has any user-labeled instances."""
+        for inst in self.instances:
+            if type(inst) == Instance:
+                return True
+        return False
+
     @property
     def predicted_instances(self) -> list[Instance]:
         """Frame instances that are predicted by a model (`PredictedInstance` objects)."""
         return [inst for inst in self.instances if type(inst) == PredictedInstance]
 
+    @property
+    def has_predicted_instances(self) -> bool:
+        """Return True if the frame has any predicted instances."""
+        for inst in self.instances:
+            if type(inst) == PredictedInstance:
+                return True
+        return False
+
     def numpy(self) -> np.ndarray:
         """Return all instances in the frame as a numpy array.
 

From 527de4e0016d0b69279565af7b7d8cd42d7baba8 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:38:36 -0700
Subject: [PATCH 18/29] Labels.user_labeled_frames

---
 sleap_io/model/labels.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/sleap_io/model/labels.py b/sleap_io/model/labels.py
index a9b979b2..09208a45 100644
--- a/sleap_io/model/labels.py
+++ b/sleap_io/model/labels.py
@@ -374,3 +374,8 @@ def remove_predictions(self, clean: bool = True):
                 tracks=True,
                 videos=False,
             )
+
+    @property
+    def user_labeled_frames(self) -> list[LabeledFrame]:
+        """Return all labeled frames with user (non-predicted) instances."""
+        return [lf for lf in self.labeled_frames if lf.has_user_instances]

From 77f7ef458c06d8068bea7496236b4ecda08fdf6d Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:38:56 -0700
Subject: [PATCH 19/29] Fix JABS

---
 sleap_io/io/jabs.py   | 3 ++-
 tests/io/test_main.py | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/sleap_io/io/jabs.py b/sleap_io/io/jabs.py
index 6aca6f7d..fd687f72 100644
--- a/sleap_io/io/jabs.py
+++ b/sleap_io/io/jabs.py
@@ -86,6 +86,7 @@ def read_labels(
     frames: List[LabeledFrame] = []
     # Video name is the pose file minus the suffix
     video_name = re.sub(r"(_pose_est_v[2-6])?\.h5", ".avi", labels_path)
+    video = Video.from_filename(video_name)
     if not skeleton:
         skeleton = JABS_DEFAULT_SKELETON
     tracks = {}
@@ -166,7 +167,7 @@ def read_labels(
                     )
                     if new_instance:
                         instances.append(new_instance)
-            frame_label = LabeledFrame(Video(video_name), frame_idx, instances)
+            frame_label = LabeledFrame(video, frame_idx, instances)
             frames.append(frame_label)
     return Labels(frames)
 
diff --git a/tests/io/test_main.py b/tests/io/test_main.py
index a9255b63..882c3295 100644
--- a/tests/io/test_main.py
+++ b/tests/io/test_main.py
@@ -55,6 +55,7 @@ def test_jabs(tmp_path, jabs_real_data_v2, jabs_real_data_v5):
     labels_single_written = load_jabs(str(tmp_path / jabs_real_data_v2))
     # Confidence field is not preserved, so just check number of labels
     assert len(labels_single) == len(labels_single_written)
+    assert len(labels_single.videos) == len(labels_single_written.videos)
     assert type(load_file(jabs_real_data_v2)) == Labels
 
     labels_multi = load_jabs(jabs_real_data_v5)
@@ -66,6 +67,7 @@ def test_jabs(tmp_path, jabs_real_data_v2, jabs_real_data_v5):
     # v5 contains all v4 and v3 data, so only need to check v5
     # Confidence field and ordering of identities is not preserved, so just check number of labels
     assert len(labels_v5_written) == len(labels_multi)
+    assert len(labels_v5_written.videos) == len(labels_multi.videos)
 
 
 def test_load_video(centered_pair_low_quality_path):

From b71646fa57d44413a21e5e5e2346d93c83f4551f Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 22:39:04 -0700
Subject: [PATCH 20/29] Tests

---
 tests/io/test_slp.py | 48 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index 766692d6..ca930b63 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -33,9 +33,10 @@
 from sleap_io.io.utils import read_hdf5_dataset
 import numpy as np
 import simplejson as json
+import pytest
 
 
-from sleap_io.io.video import ImageVideo
+from sleap_io.io.video import ImageVideo, HDF5Video, MediaVideo
 
 
 def test_read_labels(slp_typical, slp_simple_skel, slp_minimal):
@@ -242,3 +243,48 @@ def test_slp_imgvideo(tmpdir, slp_imgvideo):
     assert type(videos[0].backend) == ImageVideo
     assert len(videos[0].filename) == 2
     assert videos[0].shape is None
+
+
+def test_pkg_roundtrip(tmpdir, slp_minimal_pkg):
+    labels = read_labels(slp_minimal_pkg)
+    assert type(labels.video.backend) == HDF5Video
+    assert labels.video.shape == (1, 384, 384, 1)
+    assert labels.video.backend.embedded_frame_inds == [0]
+    assert labels.video.filename == slp_minimal_pkg
+
+    write_labels(str(tmpdir / "roundtrip.pkg.slp"), labels)
+    labels = read_labels(str(tmpdir / "roundtrip.pkg.slp"))
+    assert type(labels.video.backend) == HDF5Video
+    assert labels.video.shape == (1, 384, 384, 1)
+    assert labels.video.backend.embedded_frame_inds == [0]
+    assert labels.video.filename == str(tmpdir / "roundtrip.pkg.slp")
+
+
+@pytest.mark.parametrize("to_embed", ["user", "suggestions", "user+suggestions"])
+def test_embed(tmpdir, slp_real_data, to_embed):
+    base_labels = read_labels(slp_real_data)
+    assert type(base_labels.video.backend) == MediaVideo
+    assert (
+        base_labels.video.filename == "tests/data/videos/centered_pair_low_quality.mp4"
+    )
+    assert base_labels.video.shape == (1100, 384, 384, 1)
+    assert len(base_labels) == 10
+    assert len(base_labels.suggestions) == 10
+    assert len(base_labels.user_labeled_frames) == 5
+
+    labels_path = str(tmpdir / "labels.pkg.slp")
+    write_labels(labels_path, base_labels, embed=to_embed)
+    labels = read_labels(labels_path)
+    assert len(labels) == 10
+    assert type(labels.video.backend) == HDF5Video
+    assert labels.video.filename == labels_path
+    assert (
+        labels.video.source_video.filename
+        == "tests/data/videos/centered_pair_low_quality.mp4"
+    )
+    if to_embed == "user":
+        assert labels.video.backend.embedded_frame_inds == [0, 990, 440, 220, 770]
+    elif to_embed == "suggestions":
+        assert len(labels.video.backend.embedded_frame_inds) == 10
+    elif to_embed == "suggestions+user":
+        assert len(labels.video.backend.embedded_frame_inds) == 10

From 4e5734b7643092915e512a4b8b8d0b4005f2c05e Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Thu, 16 May 2024 23:26:24 -0700
Subject: [PATCH 21/29] Add live coverage support

---
 .gitignore      |  1 +
 CONTRIBUTING.md |  9 +++++++++
 pyproject.toml  | 24 +++++++++++++++---------
 3 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/.gitignore b/.gitignore
index f8070c49..4c9ab20a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -50,6 +50,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+lcov.info
 
 # Translations
 *.mo
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 220d37d9..e58b85ff 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -93,6 +93,15 @@ We check for coverage by parsing the outputs from `pytest` and uploading to [Cod
 
 All changes should aim to increase or maintain test coverage.
 
+### Live coverage
+
+*The following steps are based on [this guide](https://jasonstitt.com/perfect-python-live-test-coverage).*
+
+1. If you already have an environment installed, `pip install -e ."[dev]"` to make sure you have the latest dev tools (namely `pytest-watch`).
+2. Install the [Coverage Gutters extension](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) in VS Code.
+3. Open a terminal, `conda activate sleap-io` and then run `ptw` to automatically run tests. This will generate a new `lcov.info` file when it's done.
+4. Enable the coverage gutters by using **Ctrl/Cmd**+**Shift**+**P**, then **Coverage Gutters: Display Coverage**.
+
 
 ### Code style
 To standardize formatting conventions, we use [`black`](https://black.readthedocs.io/en/stable/).
diff --git a/pyproject.toml b/pyproject.toml
index 6ee831cf..0fb2c4ee 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,8 +7,7 @@ name = "sleap-io"
 authors = [
     {name = "Liezl Maree", email = "lmaree@salk.edu"},
     {name = "David Samy", email = "davidasamy@gmail.com"},
-    {name = "Talmo Pereira", email = "talmo@salk.edu"}
-]
+    {name = "Talmo Pereira", email = "talmo@salk.edu"}]
 description="Standalone utilities for working with pose data from SLEAP and other tools."
 requires-python = ">=3.7"
 keywords = ["sleap", "pose tracking", "pose estimation", "behavior"]
@@ -19,8 +18,7 @@ classifiers = [
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12"
-]
+    "Programming Language :: Python :: 3.12"]
 dependencies = [
     "numpy",
     "attrs",
@@ -31,8 +29,7 @@ dependencies = [
     "simplejson",
     "imageio",
     "imageio-ffmpeg",
-    "av"
-]
+    "av"]
 dynamic = ["version", "readme"]
 
 [tool.setuptools.dynamic]
@@ -43,6 +40,7 @@ readme = {file = ["README.md"], content-type="text/markdown"}
 dev = [
     "pytest",
     "pytest-cov",
+    "pytest-watch",
     "black",
     "pydocstyle",
     "toml",
@@ -52,16 +50,24 @@ dev = [
     "mkdocs-jupyter",
     "mkdocstrings[python]>=0.18",
     "mkdocs-gen-files",
-    "mkdocs-literate-nav"
-]
+    "mkdocs-literate-nav"]
 
 [project.urls]
-Homepage = "https://sleap.ai"
+Homepage = "https://io.sleap.ai"
 Repository = "https://github.com/talmolab/sleap-io"
 
+[tool.setuptools.packages.find]
+exclude = ["site"]
+
 [tool.black]
 line-length = 88
 
 [pydocstyle]
 convention = "google"
 match-dir = "sleap_io"
+
+[tool.coverage.run]
+source = ["livecov"]
+
+[tool.pytest.ini_options]
+addopts = "--cov sleap_io --cov-report=lcov:lcov.info --cov-report=term"

From 0614541be316486f94eafccdf747615224b3069e Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Tue, 4 Jun 2024 21:21:21 -0400
Subject: [PATCH 22/29] Expose high level embedding

---
 sleap_io/io/main.py      | 10 ++++++++--
 sleap_io/model/labels.py | 15 +++++++++++++--
 2 files changed, 21 insertions(+), 4 deletions(-)

diff --git a/sleap_io/io/main.py b/sleap_io/io/main.py
index 23720b86..32432ac4 100644
--- a/sleap_io/io/main.py
+++ b/sleap_io/io/main.py
@@ -19,14 +19,20 @@ def load_slp(filename: str) -> Labels:
     return slp.read_labels(filename)
 
 
-def save_slp(labels: Labels, filename: str):
+def save_slp(
+    labels: Labels, filename: str, embed: str | list[tuple[Video, int]] | None = None
+):
     """Save a SLEAP dataset to a `.slp` file.
 
     Args:
         labels: A SLEAP `Labels` object (see `load_slp`).
         filename: Path to save labels to ending with `.slp`.
+        embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, `"source"` or
+            list of tuples of `(video, frame_idx)` specifying the frames to embed. If
+            `"source"` is specified, no images will be embedded and the source video
+            will be restored if available.
     """
-    return slp.write_labels(filename, labels)
+    return slp.write_labels(filename, labels, embed=embed)
 
 
 def load_nwb(filename: str) -> Labels:
diff --git a/sleap_io/model/labels.py b/sleap_io/model/labels.py
index 09208a45..5a047f79 100644
--- a/sleap_io/model/labels.py
+++ b/sleap_io/model/labels.py
@@ -279,7 +279,13 @@ def find(
 
         return results
 
-    def save(self, filename: str, format: Optional[str] = None, **kwargs):
+    def save(
+        self,
+        filename: str,
+        format: Optional[str] = None,
+        embed: str | list[tuple[Video, int]] | None = None,
+        **kwargs,
+    ):
         """Save labels to file in specified format.
 
         Args:
@@ -287,10 +293,15 @@ def save(self, filename: str, format: Optional[str] = None, **kwargs):
             format: The format to save the labels in. If `None`, the format will be
                 inferred from the file extension. Available formats are "slp", "nwb",
                 "labelstudio", and "jabs".
+            embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, `"source"` or
+                list of tuples of `(video, frame_idx)` specifying the frames to embed.
+                If `"source"` is specified, no images will be embedded and the source
+                video will be restored if available. This argument is only valid for the
+                SLP backend.
         """
         from sleap_io import save_file
 
-        save_file(self, filename, format=format, **kwargs)
+        save_file(self, filename, format=format, embed=embed, **kwargs)
 
     def clean(
         self,

From 38948371c218d4ef00544ee607b219a8ea1acac2 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Tue, 4 Jun 2024 21:21:57 -0400
Subject: [PATCH 23/29] Separate replace video and support restoring source

---
 sleap_io/io/slp.py         | 98 ++++++++++++++++++++------------------
 sleap_io/model/labels.py   | 27 +++++++++++
 tests/io/test_slp.py       | 24 ++++++++++
 tests/model/test_labels.py | 14 ++++++
 4 files changed, 116 insertions(+), 47 deletions(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 3ce4258d..73d394fa 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -307,7 +307,7 @@ def embed_video(
 def embed_frames(
     labels_path: str,
     labels: Labels,
-    to_embed: list[tuple[Video, int]],
+    embed: list[tuple[Video, int]],
     image_format: str = "png",
 ):
     """Embed frames in a SLEAP labels file.
@@ -315,8 +315,7 @@ def embed_frames(
     Args:
         labels_path: A string path to the SLEAP labels file.
         labels: A `Labels` object to embed in the labels file.
-        to_embed: A list of tuples of `(video, frame_idx)` specifying the frames to
-            embed.
+        embed: A list of tuples of `(video, frame_idx)` specifying the frames to embed.
         image_format: The image format to use for embedding. Valid formats are "png"
             (the default), "jpg" or "hdf5".
 
@@ -325,7 +324,7 @@ def embed_frames(
         and `Labels` objects in place.
     """
     to_embed_by_video = {}
-    for video, frame_idx in to_embed:
+    for video, frame_idx in embed:
         if video not in to_embed_by_video:
             to_embed_by_video[video] = []
         to_embed_by_video[video].append(frame_idx)
@@ -344,70 +343,73 @@ def embed_frames(
         labels.videos[video_ind] = embedded_video
         replaced_videos[video] = embedded_video
 
-    # Update the labeled frames with the new videos.
-    for lf in labels:
-        if lf.video in replaced_videos:
-            lf.video = replaced_videos[lf.video]
-
-    # Update suggestions with the new videos.
-    for sf in labels.suggestions:
-        if sf.video in replaced_videos:
-            sf.video = replaced_videos[sf.video]
+    if len(replaced_videos) > 0:
+        labels.replace_videos(video_map=replaced_videos)
 
 
 def embed_videos(
-    labels_path: str, labels: Labels, to_embed: str | list[tuple[Video, int]]
+    labels_path: str, labels: Labels, embed: str | list[tuple[Video, int]]
 ):
     """Embed videos in a SLEAP labels file.
 
     Args:
         labels_path: A string path to the SLEAP labels file to save.
         labels: A `Labels` object to save.
-        to_embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, or list of
-            tuples of `(video, frame_idx)` specifying the frames to embed.
+        embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, `"source"` or
+            list of tuples of `(video, frame_idx)` specifying the frames to embed. If
+            `"source"` is specified, no images will be embedded and the source video
+            will be restored if available.
     """
 
-    if to_embed == "user":
-        to_embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
-    elif to_embed == "suggestions":
-        to_embed = [(sf.video, sf.frame_idx) for sf in labels.suggestions]
-    elif to_embed == "user+suggestions":
-        to_embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
-        to_embed += [(sf.video, sf.frame_idx) for sf in labels.suggestions]
-    elif isinstance(to_embed, list):
-        to_embed = to_embed
+    if embed == "user":
+        embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
+    elif embed == "suggestions":
+        embed = [(sf.video, sf.frame_idx) for sf in labels.suggestions]
+    elif embed == "user+suggestions":
+        embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
+        embed += [(sf.video, sf.frame_idx) for sf in labels.suggestions]
+    elif embed == "source":
+        embed = []
+    elif isinstance(embed, list):
+        embed = embed
     else:
-        raise ValueError(f"Invalid value for to_embed: {to_embed}")
+        raise ValueError(f"Invalid value for embed: {embed}")
 
-    embed_frames(labels_path, labels, to_embed)
+    embed_frames(labels_path, labels, embed)
 
 
-def write_videos(labels_path: str, videos: list[Video]):
+def write_videos(labels_path: str, videos: list[Video], restore_source: bool = False):
     """Write video metadata to a SLEAP labels file.
 
     Args:
         labels_path: A string path to the SLEAP labels file.
         videos: A list of `Video` objects to store the metadata for.
+        restore_source: If `True`, restore source videos if available and will not
+            re-embed the embedded images. If `False` (the default), will re-embed images
+            that were previously embedded.
     """
     video_jsons = []
     for video_ind, video in enumerate(videos):
 
         if type(video.backend) == HDF5Video and video.backend.has_embedded_images:
-            # If the video has embedded images, embed them images again if we haven't
-            # already.
-            already_embedded = False
-            if Path(labels_path).exists():
-                with h5py.File(labels_path, "r") as f:
-                    already_embedded = f"video{video_ind}/video" in f
-
-            if not already_embedded:
-                video = embed_video(
-                    labels_path,
-                    video,
-                    group=f"video{video_ind}",
-                    frame_inds=video.backend.source_inds,
-                    image_format=video.backend.image_format,
-                )
+            if restore_source:
+                video = video.source_video
+            else:
+                # If the video has embedded images, embed them images again if we haven't
+                # already.
+                already_embedded = False
+                if Path(labels_path).exists():
+                    with h5py.File(labels_path, "r") as f:
+                        already_embedded = f"video{video_ind}/video" in f
+
+                if not already_embedded:
+                    video = embed_video(
+                        labels_path,
+                        video,
+                        group=f"video{video_ind}",
+                        frame_inds=video.backend.source_inds,
+                        image_format=video.backend.image_format,
+                    )
 
         video_json = video_to_dict(video)
 
@@ -1022,15 +1024,17 @@ def write_labels(
     Args:
         labels_path: A string path to the SLEAP labels file to save.
         labels: A `Labels` object to save.
-        embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, or list of tuples
-            of `(video, frame_idx)` specifying the frames to embed. If `None`, no frames
-            will be embedded. Existing embedded videos will be re-saved regardless.
+        embed: One of `"user"`, `"suggestions"`, `"user+suggestions"`, `"source"`,
+            `None` or list of tuples of `(video, frame_idx)` specifying the frames to
+            embed. If `"source"` is specified, no images will be embedded and the source
+            video will be restored if available. If `None` is specified (the default),
+            existing embedded images will be re-embedded.
     """
     if Path(labels_path).exists():
         Path(labels_path).unlink()
     if embed is not None:
         embed_videos(labels_path, labels, embed)
-    write_videos(labels_path, labels.videos)
+    write_videos(labels_path, labels.videos, restore_source=(embed == "source"))
     write_tracks(labels_path, labels.tracks)
     write_suggestions(labels_path, labels.suggestions, labels.videos)
     write_metadata(labels_path, labels)
diff --git a/sleap_io/model/labels.py b/sleap_io/model/labels.py
index 5a047f79..841c272a 100644
--- a/sleap_io/model/labels.py
+++ b/sleap_io/model/labels.py
@@ -390,3 +390,30 @@ def remove_predictions(self, clean: bool = True):
     def user_labeled_frames(self) -> list[LabeledFrame]:
         """Return all labeled frames with user (non-predicted) instances."""
         return [lf for lf in self.labeled_frames if lf.has_user_instances]
+
+    def replace_videos(
+        self,
+        old_videos: list[Video] | None = None,
+        new_videos: list[Video] | None = None,
+        video_map: dict[Video, Video] | None = None,
+    ):
+        """Replace videos and update all references.
+
+        Args:
+            old_videos: List of videos to be replaced.
+            new_videos: List of videos to replace with.
+            video_map: Alternative input of dictionary where keys are the old videos and
+                values are the new videos.
+        """
+        if video_map is None:
+            video_map = {o: n for o, n in zip(old_videos, new_videos)}
+
+        # Update the labeled frames with the new videos.
+        for lf in self.labeled_frames:
+            if lf.video in video_map:
+                lf.video = video_map[lf.video]
+
+        # Update suggestions with the new videos.
+        for sf in self.suggestions:
+            if sf.video in video_map:
+                sf.video = video_map[sf.video]
diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index ca930b63..827476d3 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -288,3 +288,27 @@ def test_embed(tmpdir, slp_real_data, to_embed):
         assert len(labels.video.backend.embedded_frame_inds) == 10
     elif to_embed == "suggestions+user":
         assert len(labels.video.backend.embedded_frame_inds) == 10
+
+
+def test_embed_two_rounds(tmpdir, slp_real_data):
+    base_labels = read_labels(slp_real_data)
+    labels_path = str(tmpdir / "labels.pkg.slp")
+    write_labels(labels_path, base_labels, embed="user")
+    labels = read_labels(labels_path)
+
+    assert labels.video.backend.embedded_frame_inds == [0, 990, 440, 220, 770]
+
+    labels2_path = str(tmpdir / "labels2.pkg.slp")
+    write_labels(labels2_path, labels)
+    labels2 = read_labels(labels2_path)
+    assert (
+        labels2.video.source_video.filename
+        == "tests/data/videos/centered_pair_low_quality.mp4"
+    )
+    assert labels2.video.backend.embedded_frame_inds == [0, 990, 440, 220, 770]
+
+    labels3_path = str(tmpdir / "labels3.slp")
+    write_labels(labels3_path, labels, embed="source")
+    labels3 = read_labels(labels3_path)
+    assert labels3.video.filename == "tests/data/videos/centered_pair_low_quality.mp4"
+    assert type(labels3.video.backend) == MediaVideo
diff --git a/tests/model/test_labels.py b/tests/model/test_labels.py
index 0060a220..52ced7c6 100644
--- a/tests/model/test_labels.py
+++ b/tests/model/test_labels.py
@@ -258,3 +258,17 @@ def test_labels_remove_predictions(slp_real_data):
     labels.remove_predictions(clean=True)
     assert len(labels) == 5
     assert sum([len(lf.predicted_instances) for lf in labels]) == 0
+
+
+def test_replace_videos(slp_real_data):
+    labels = load_slp(slp_real_data)
+    assert labels.video.filename == "tests/data/videos/centered_pair_low_quality.mp4"
+    labels.replace_videos(
+        old_videos=[labels.video], new_videos=[Video.from_filename("fake.mp4")]
+    )
+
+    for lf in labels:
+        assert lf.video.filename == "fake.mp4"
+
+    for sf in labels.suggestions:
+        assert sf.video.filename == "fake.mp4"

From a76680a5c3b37350cea8c8e30f12bf1fdd923e53 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 10:27:50 +0100
Subject: [PATCH 24/29] Lint

---
 sleap_io/io/slp.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sleap_io/io/slp.py b/sleap_io/io/slp.py
index 14941a57..4a1f5f85 100644
--- a/sleap_io/io/slp.py
+++ b/sleap_io/io/slp.py
@@ -360,7 +360,6 @@ def embed_videos(
             `"source"` is specified, no images will be embedded and the source video
             will be restored if available.
     """
-
     if embed == "user":
         embed = [(lf.video, lf.frame_idx) for lf in labels.user_labeled_frames]
     elif embed == "suggestions":

From 64f88e5ef2a9e0d2323b450b4aab79ac0ab69831 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 10:35:17 +0100
Subject: [PATCH 25/29] Add Video(filename) syntactic sugar

---
 sleap_io/model/video.py   | 5 +++++
 tests/model/test_video.py | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/sleap_io/model/video.py b/sleap_io/model/video.py
index 04332e28..2c547804 100644
--- a/sleap_io/model/video.py
+++ b/sleap_io/model/video.py
@@ -47,6 +47,11 @@ class Video:
 
     EXTS = MediaVideo.EXTS + HDF5Video.EXTS
 
+    def __attrs_post_init__(self):
+        """Post init syntactic sugar."""
+        if self.backend is None and self.exists():
+            self.open()
+
     @classmethod
     def from_filename(
         cls,
diff --git a/tests/model/test_video.py b/tests/model/test_video.py
index 0ccd13c2..4770fc96 100644
--- a/tests/model/test_video.py
+++ b/tests/model/test_video.py
@@ -57,8 +57,8 @@ def test_video_exists(centered_pair_low_quality_video, centered_pair_frame_paths
 
 def test_video_open_close(centered_pair_low_quality_path):
     video = Video(centered_pair_low_quality_path)
-    assert video.is_open is False
-    assert video.backend is None
+    assert video.is_open
+    assert type(video.backend) == MediaVideo
 
     img = video[0]
     assert img.shape == (384, 384, 1)

From 8932247b3eb8adf3dc82cc01b298f418d371c9f3 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 10:59:51 +0100
Subject: [PATCH 26/29] Coverage

---
 tests/model/test_labeled_frame.py | 50 ++++++++++++++++++++++++++++++-
 tests/model/test_video.py         |  1 +
 2 files changed, 50 insertions(+), 1 deletion(-)

diff --git a/tests/model/test_labeled_frame.py b/tests/model/test_labeled_frame.py
index b542124e..42f40428 100644
--- a/tests/model/test_labeled_frame.py
+++ b/tests/model/test_labeled_frame.py
@@ -1,7 +1,7 @@
 """Tests for methods in sleap_io.model.labeled_frame file."""
 
 from numpy.testing import assert_equal
-from sleap_io import Video, Skeleton, Instance, PredictedInstance
+from sleap_io import Video, Skeleton, Instance, PredictedInstance, Track
 from sleap_io.model.labeled_frame import LabeledFrame
 import numpy as np
 
@@ -28,6 +28,9 @@ def test_labeled_frame():
     # Test LabeledFrame.__getitem__ method
     assert lf[0] == inst
 
+    assert lf.has_predicted_instances
+    assert lf.has_user_instances
+
 
 def test_remove_predictions():
     """Test removing predictions from `LabeledFrame`."""
@@ -43,6 +46,8 @@ def test_remove_predictions():
 
     assert len(lf) == 2
     assert len(lf.predicted_instances) == 1
+    assert lf.has_predicted_instances
+    assert lf.has_user_instances
 
     # Remove predictions
     lf.remove_predictions()
@@ -51,6 +56,8 @@ def test_remove_predictions():
     assert len(lf.predicted_instances) == 0
     assert type(lf[0]) == Instance
     assert_equal(lf.numpy(), [[[0, 1], [2, 3]]])
+    assert not lf.has_predicted_instances
+    assert lf.has_user_instances
 
 
 def test_remove_empty_instances():
@@ -75,3 +82,44 @@ def test_remove_empty_instances():
     assert len(lf) == 1
     assert type(lf[0]) == Instance
     assert_equal(lf.numpy(), [[[0, 1], [2, 3]]])
+
+
+def test_labeled_frame_image(centered_pair_low_quality_path):
+    video = Video.from_filename(centered_pair_low_quality_path)
+    lf = LabeledFrame(video=video, frame_idx=0)
+    assert_equal(lf.image, video[0])
+
+
+def test_labeled_frame_unused_predictions():
+    video = Video("test.mp4")
+    skel = Skeleton(["A", "B"])
+    track = Track("trk")
+
+    lf1 = LabeledFrame(video=video, frame_idx=0)
+    lf1.instances.append(
+        Instance.from_numpy([[0, 0], [0, 0]], skeleton=skel, track=track)
+    )
+    lf1.instances.append(
+        PredictedInstance.from_numpy(
+            [[0, 0], [0, 0]], [1, 1], 1, skeleton=skel, track=track
+        )
+    )
+    lf1.instances.append(
+        PredictedInstance.from_numpy([[1, 1], [1, 1]], [1, 1], 1, skeleton=skel)
+    )
+
+    assert len(lf1.unused_predictions) == 1
+    assert (lf1.unused_predictions[0].numpy() == 1).all()
+
+    lf2 = LabeledFrame(video=video, frame_idx=1)
+    lf2.instances.append(
+        PredictedInstance.from_numpy([[0, 0], [0, 0]], [1, 1], 1, skeleton=skel)
+    )
+    lf2.instances.append(Instance.from_numpy([[0, 0], [0, 0]], skeleton=skel))
+    lf2.instances[-1].from_predicted = lf2.instances[-2]
+    lf2.instances.append(
+        PredictedInstance.from_numpy([[1, 1], [1, 1]], [1, 1], 1, skeleton=skel)
+    )
+
+    assert len(lf2.unused_predictions) == 1
+    assert (lf2.unused_predictions[0].numpy() == 1).all()
diff --git a/tests/model/test_video.py b/tests/model/test_video.py
index 4770fc96..03a0bdba 100644
--- a/tests/model/test_video.py
+++ b/tests/model/test_video.py
@@ -20,6 +20,7 @@ def test_video_from_filename(centered_pair_low_quality_path):
     test_video = Video.from_filename(centered_pair_low_quality_path)
     assert test_video.filename == centered_pair_low_quality_path
     assert test_video.shape == (1100, 384, 384, 1)
+    assert len(test_video) == 1100
     assert type(test_video.backend) == MediaVideo
 
 

From eff967694959630b7a3c298ccfb31a26f9534ab4 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 11:14:13 +0100
Subject: [PATCH 27/29] Windows test fix

---
 tests/io/test_slp.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index 4a78f3ad..65afbd30 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -37,7 +37,7 @@
 import numpy as np
 import simplejson as json
 import pytest
-
+from pathlib import Path
 
 from sleap_io.io.video import ImageVideo, HDF5Video, MediaVideo
 
@@ -285,7 +285,8 @@ def test_embed(tmpdir, slp_real_data, to_embed):
     base_labels = read_labels(slp_real_data)
     assert type(base_labels.video.backend) == MediaVideo
     assert (
-        base_labels.video.filename == "tests/data/videos/centered_pair_low_quality.mp4"
+        Path(base_labels.video.filename).as_posix()
+        == "tests/data/videos/centered_pair_low_quality.mp4"
     )
     assert base_labels.video.shape == (1100, 384, 384, 1)
     assert len(base_labels) == 10
@@ -299,7 +300,7 @@ def test_embed(tmpdir, slp_real_data, to_embed):
     assert type(labels.video.backend) == HDF5Video
     assert labels.video.filename == labels_path
     assert (
-        labels.video.source_video.filename
+        Path(labels.video.source_video.filename).as_posix()
         == "tests/data/videos/centered_pair_low_quality.mp4"
     )
     if to_embed == "user":
@@ -322,7 +323,7 @@ def test_embed_two_rounds(tmpdir, slp_real_data):
     write_labels(labels2_path, labels)
     labels2 = read_labels(labels2_path)
     assert (
-        labels2.video.source_video.filename
+        Path(labels2.video.source_video.filename).as_posix()
         == "tests/data/videos/centered_pair_low_quality.mp4"
     )
     assert labels2.video.backend.embedded_frame_inds == [0, 990, 440, 220, 770]
@@ -330,5 +331,8 @@ def test_embed_two_rounds(tmpdir, slp_real_data):
     labels3_path = str(tmpdir / "labels3.slp")
     write_labels(labels3_path, labels, embed="source")
     labels3 = read_labels(labels3_path)
-    assert labels3.video.filename == "tests/data/videos/centered_pair_low_quality.mp4"
+    assert (
+        Path(labels3.video.filename).as_posix()
+        == "tests/data/videos/centered_pair_low_quality.mp4"
+    )
     assert type(labels3.video.backend) == MediaVideo

From 5eee08303f37a6621a8918698afc86188a4c28a5 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 11:32:50 +0100
Subject: [PATCH 28/29] Windows test fix again

---
 tests/io/test_slp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index 65afbd30..33bcd939 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -277,7 +277,10 @@ def test_pkg_roundtrip(tmpdir, slp_minimal_pkg):
     assert type(labels.video.backend) == HDF5Video
     assert labels.video.shape == (1, 384, 384, 1)
     assert labels.video.backend.embedded_frame_inds == [0]
-    assert labels.video.filename == str(tmpdir / "roundtrip.pkg.slp")
+    assert (
+        Path(labels.video.filename).as_posix()
+        == (tmpdir / "roundtrip.pkg.slp").as_posix()
+    )
 
 
 @pytest.mark.parametrize("to_embed", ["user", "suggestions", "user+suggestions"])
@@ -293,12 +296,12 @@ def test_embed(tmpdir, slp_real_data, to_embed):
     assert len(base_labels.suggestions) == 10
     assert len(base_labels.user_labeled_frames) == 5
 
-    labels_path = str(tmpdir / "labels.pkg.slp")
+    labels_path = Path(tmpdir / "labels.pkg.slp").as_posix()
     write_labels(labels_path, base_labels, embed=to_embed)
     labels = read_labels(labels_path)
     assert len(labels) == 10
     assert type(labels.video.backend) == HDF5Video
-    assert labels.video.filename == labels_path
+    assert Path(labels.video.filename).as_posix() == labels_path
     assert (
         Path(labels.video.source_video.filename).as_posix()
         == "tests/data/videos/centered_pair_low_quality.mp4"

From d2a1415d0d9bf24ba944837719cfcd169c257848 Mon Sep 17 00:00:00 2001
From: Talmo Pereira <talmo@salk.edu>
Date: Wed, 5 Jun 2024 11:40:21 +0100
Subject: [PATCH 29/29] Fix test

---
 tests/io/test_slp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/io/test_slp.py b/tests/io/test_slp.py
index 33bcd939..7d229bc4 100644
--- a/tests/io/test_slp.py
+++ b/tests/io/test_slp.py
@@ -279,7 +279,7 @@ def test_pkg_roundtrip(tmpdir, slp_minimal_pkg):
     assert labels.video.backend.embedded_frame_inds == [0]
     assert (
         Path(labels.video.filename).as_posix()
-        == (tmpdir / "roundtrip.pkg.slp").as_posix()
+        == Path(tmpdir / "roundtrip.pkg.slp").as_posix()
     )