Merge pull request #1 from bioio-devs/bioio-nd2

feature/nd2-reader

.github/workflows/ci.yml

@@ -37,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9, 3.10, 3.11]
+        python-version: [3.9, "3.10", 3.11]
         os: [ubuntu-latest, macOS-latest, windows-latest]
 
     steps:
@@ -53,6 +53,15 @@ jobs:
       run: |
         python -m pip install --upgrade pip
         pip install .[test]
+    - uses: actions/cache@v2
+      id: cache
+      with:
+        path: bioio_nd2/tests/resources
+        key: ${{ hashFiles('scripts/TEST_RESOURCES_HASH.txt') }}
+    - name: Download Test Resources
+      if: steps.cache.outputs.cache-hit != 'true'
+      run: |
+          python scripts/download_test_resources.py --debug
     - name: Run Tests
       run: just test
     - name: Upload Codecov

.gitignore

@@ -6,6 +6,9 @@ __pycache__/
 # C extensions
 *.so
 
+# Test files 
+bioio_nd2/tests/resources
+
 # Distribution / packaging
 .Python
 env/

.pre-commit-config.yaml

@@ -29,3 +29,4 @@ repos:
     rev: v1.4.1
     hooks:
       - id: mypy
+        additional_dependencies: [types-PyYAML>=6.0.12.9]

CONTRIBUTING.md

@@ -25,7 +25,7 @@ Ready to contribute? Here's how to set up `bioio-nd2` for local development.
 
     ```bash
     cd bioio-nd2/
-    just install
+    just setup-dev
     ```
 
 4. Create a branch for local development:

Justfile

@@ -24,6 +24,12 @@ clean:
 install:
 	pip install -e .[lint,test,docs]
 
+# install dependencies, setup pre-commit, download test resources
+setup-dev:
+	just install
+	pre-commit install
+	python scripts/download_test_resources.py
+
 # lint, format, and check all files
 lint:
 	pre-commit run --all-files

bioio_nd2/__init__.py

@@ -12,7 +12,7 @@
 __author__ = "bioio-devs"
 __email__ = "[email protected]"
 
-from .reader_metadata import Reader
+from .reader import Reader
 from .reader_metadata import ReaderMetadata
 
 __all__ = ["Reader", "ReaderMetadata"]

bioio_nd2/reader.py

@@ -1,109 +1,94 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from typing import Any, Optional, Tuple
-
-from bioio_base.dimensions import Dimensions
-from bioio_base.reader import Reader as BaseReader
+from typing import Any, Dict, Tuple
 
+import nd2
 import xarray as xr
+from bioio_base import constants, exceptions, io, reader, types
+from fsspec.implementations.local import LocalFileSystem
 from fsspec.spec import AbstractFileSystem
 
 ###############################################################################
 
 
-class Reader(BaseReader):
-    """
-    The main class of each reader plugin. This class is subclass
-    of the abstract class reader (BaseReader) in bioio-base.
+class Reader(reader.Reader):
+    """Read NIS-Elements files using the Nikon nd2 SDK.
+
+    This reader requires `nd2` to be installed in the environment.
 
     Parameters
     ----------
-    image: Any
-        Some type of object to read and follow the Reader specification.
+    image : Path or str
+        path to file
     fs_kwargs: Dict[str, Any]
         Any specific keyword arguments to pass down to the fsspec created filesystem.
         Default: {}
 
-    Notes
-    -----
-    It is up to the implementer of the Reader to decide which types they would like to
-    accept (certain readers may not support buffers for example).
-
+    Raises
+    ------
+    exceptions.UnsupportedFileFormatError
+        If the file is not supported by ND2.
     """
-    _xarray_dask_data: Optional["xr.DataArray"] = None
-    _xarray_data: Optional["xr.DataArray"] = None
-    _mosaic_xarray_dask_data: Optional["xr.DataArray"] = None
-    _mosaic_xarray_data: Optional["xr.DataArray"] = None
-    _dims: Optional[Dimensions] = None
-    _metadata: Optional[Any] = None
-    _scenes: Optional[Tuple[str, ...]] = None
-    _current_scene_index: int = 0
-    # Do not provide default value because
-    # they may not need to be used by your reader (i.e. input param is an array)
-    _fs: "AbstractFileSystem"
-    _path: str
-
-    # Required Methods
-
-    def __init__(image: Any, **kwargs: Any):
-        """
-        Store / cache certain parameters required for later reading.
-
-        Try not to read the image into memory here.
-        """
-        raise NotImplementedError()
 
     @staticmethod
-    def _is_supported_image(fs: "AbstractFileSystem", path: str, **kwargs: Any) -> bool:
-        """
-        Perform a check to determine if the object(s) or path(s) provided as
-        parameters are supported by this reader.
-        """
-        raise NotImplementedError()
+    def _is_supported_image(fs: AbstractFileSystem, path: str, **kwargs: Any) -> bool:
+        return nd2.is_supported_file(path, fs.open)
+
+    def __init__(self, image: types.PathLike, fs_kwargs: Dict[str, Any] = {}):
+        self._fs, self._path = io.pathlike_to_fs(
+            image,
+            enforce_exists=True,
+            fs_kwargs=fs_kwargs,
+        )
+        # Catch non-local file system
+        if not isinstance(self._fs, LocalFileSystem):
+            raise ValueError(
+                f"Cannot read ND2 from non-local file system. "
+                f"Received URI: {self._path}, which points to {type(self._fs)}."
+            )
+
+        if not self._is_supported_image(self._fs, self._path):
+            raise exceptions.UnsupportedFileFormatError(
+                self.__class__.__name__, self._path
+            )
 
     @property
     def scenes(self) -> Tuple[str, ...]:
-        """
-        Return the list of available scenes for the file using the
-        cached parameters stored to the object in the __init__.
-        """
-        raise NotImplementedError()
-
-    def _read_delayed(self) -> "xr.DataArray":
-        """
-        Return an xarray DataArray filled with a delayed dask array, coordinate planes,
-        and any metadata stored in the attrs.
-
-        Metadata should be labelled with one of the bioio-base constants.
-        """
-        raise NotImplementedError()
-
-    def _read_immediate(self) -> "xr.DataArray":
-        """
-        Return an xarray DataArray filled with an in-memory numpy ndarray,
-        coordinate planes, and any metadata stored in the attrs.
-
-        Metadata should be labelled with one of the bioio-base constants.
-        """
-        raise NotImplementedError()
-
-    # Optional Methods
-
-    def _get_stitched_dask_mosaic(self) -> "xr.DataArray":
-        """
-        If your file returns an `M` dimension for "Mosiac Tile",
-        this function should stitch and return the stitched data as
-        an xarray DataArray both operating against the original delayed array
-        and returning a delayed array.
-        """
-        return super()._get_stitched_dask_mosaic()
+        with nd2.ND2File(self._path) as rdr:
+            return tuple(rdr._position_names())
+
+    def _read_delayed(self) -> xr.DataArray:
+        return self._xarr_reformat(delayed=True)
+
+    def _read_immediate(self) -> xr.DataArray:
+        return self._xarr_reformat(delayed=False)
+
+    def _xarr_reformat(self, delayed: bool) -> xr.DataArray:
+        with nd2.ND2File(self._path) as rdr:
+            xarr = rdr.to_xarray(
+                delayed=delayed, squeeze=False, position=self.current_scene_index
+            )
+            xarr.attrs[constants.METADATA_UNPROCESSED] = xarr.attrs.pop("metadata")
+            if self.current_scene_index is not None:
+                xarr.attrs[constants.METADATA_UNPROCESSED][
+                    "frame"
+                ] = rdr.frame_metadata(self.current_scene_index)
+        return xarr.isel({nd2.AXIS.POSITION: 0}, missing_dims="ignore")
 
-    def _get_stitched_mosaic(self) -> "xr.DataArray":
+    @property
+    def physical_pixel_sizes(self) -> types.PhysicalPixelSizes:
         """
-        If your file returns an `M` dimension for "Mosiac Tile",
-        this function should stitch and return the stitched data as
-        an xarray DataArray both operating against the original in-memory array
-        and returning a in-memory array.
+        Returns
+        -------
+        sizes: PhysicalPixelSizes
+            Using available metadata, the floats representing physical pixel sizes for
+            dimensions Z, Y, and X.
+
+        Notes
+        -----
+        We currently do not handle unit attachment to these values. Please see the file
+        metadata for unit information.
         """
-        return super()._get_stitched_mosaic()
+        with nd2.ND2File(self._path) as rdr:
+            return types.PhysicalPixelSizes(*rdr.voxel_size()[::-1])

bioio_nd2/reader_metadata.py

@@ -12,16 +12,16 @@ class ReaderMetadata(bioio_base.reader_metadata.ReaderMetadata):
     """
     Notes
     -----
-    Defines metadata for the reader itself (not the image read), 
+    Defines metadata for the reader itself (not the image read),
     such as supported file extensions.
     """
+
     @staticmethod
     def get_supported_extensions() -> List[str]:
         """
         Return a list of file extensions this plugin supports reading.
         """
-        raise NotImplementedError()
-        # return ["ext", "extn"]
+        return [".nd2"]
 
     @staticmethod
     def get_reader() -> bioio_base.reader.Reader:

bioio_nd2/tests/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

bioio_nd2/tests/conftest.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import pathlib
+
+LOCAL_RESOURCES_DIR = pathlib.Path(__file__).parent / "resources"