Skip to content

Commit

Permalink
use Path
Browse files Browse the repository at this point in the history
  • Loading branch information
@leoschwarz
leoschwarz committed Jul 1, 2024
1 parent d7f3138 commit 3c150e2
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 37 deletions.
17 changes: 9 additions & 8 deletions src/depiction/parallel_ops/write_spectra_parallel.py
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
from __future__ import annotations

import contextlib
import functools
import os
from typing import Callable, Any, TYPE_CHECKING
from pathlib import Path
from tempfile import TemporaryDirectory
from typing import Callable, Any, TYPE_CHECKING

from depiction.parallel_ops import ReadSpectraParallel
from depiction.persistence import (
Expand Down Expand Up @@ -54,7 +55,7 @@ def map_chunked_to_files(
"""
with TemporaryDirectory() as work_directory:
split_modes_and_paths = self._get_split_modes_and_paths(
work_directory=work_directory,
work_directory=Path(work_directory),
spectra_indices=spectra_indices,
read_file=read_file,
write_files=write_files,
Expand Down Expand Up @@ -97,7 +98,7 @@ def op(
# TODO also it's currently untested
# copy the relevant spectra to a temporary file
with TemporaryDirectory() as work_directory:
split_input_file = os.path.join(work_directory, "tmp_file.imzML")
split_input_file = Path(work_directory) / "tmp_file.imzML"
with ImzmlWriteFile(
split_input_file,
imzml_mode=reader.imzml_mode,
Expand All @@ -120,11 +121,11 @@ def op(

def _get_split_modes_and_paths(
self,
work_directory: str,
work_directory: Path,
read_file: ImzmlReadFile,
write_files: list[ImzmlWriteFile],
spectra_indices: NDArray[int] | None,
) -> list[tuple[ImzmlModeEnum, list[str]]]:
) -> list[tuple[ImzmlModeEnum, list[Path]]]:
# determine the number of tasks
if spectra_indices is not None:
n_tasks = self._config.get_splits_count(n_items=len(spectra_indices))
Expand All @@ -135,7 +136,7 @@ def _get_split_modes_and_paths(
return [
(
write_file.imzml_mode,
[os.path.join(work_directory, f"chunk_f{i_file}_t{i_task}.imzML") for i_task in range(n_tasks)],
[work_directory / f"chunk_f{i_file}_t{i_task}.imzML" for i_task in range(n_tasks)],
)
for i_file, write_file in enumerate(write_files)
]
Expand All @@ -150,7 +151,7 @@ def _write_transformed_chunked_operation(
| Callable[[ImzmlReader, list[int], list[ImzmlWriteFile], ...], None]
),
open_write_files: bool,
split_modes_and_paths: list[tuple[ImzmlModeEnum, list[str]]],
split_modes_and_paths: list[tuple[ImzmlModeEnum, list[Path]]],
) -> None:
"""Performs the operation for a chunk of spectra. To be called in a parallel context.
:param reader: the reader to read the spectra from
Expand Down
58 changes: 29 additions & 29 deletions tests/unit/parallel_ops/test_write_spectra_parallel.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,17 +20,17 @@ def setUp(self) -> None:
(
ImzmlModeEnum.CONTINUOUS,
[
"/dev/null/mock/continuous_0.imzML",
"/dev/null/mock/continuous_1.imzML",
"/dev/null/mock/continuous_2.imzML",
Path("/dev/null/mock/continuous_0.imzML"),
Path("/dev/null/mock/continuous_1.imzML"),
Path("/dev/null/mock/continuous_2.imzML"),
],
),
(
ImzmlModeEnum.PROCESSED,
[
"/dev/null/mock/processed_0.imzML",
"/dev/null/mock/processed_1.imzML",
"/dev/null/mock/processed_2.imzML",
Path("/dev/null/mock/processed_0.imzML"),
Path("/dev/null/mock/processed_1.imzML"),
Path("/dev/null/mock/processed_2.imzML"),
],
),
]
Expand Down Expand Up @@ -66,7 +66,7 @@ def test_map_chunked_to_files(
)

mock_get_split_modes_and_paths.assert_called_once_with(
work_directory="/dev/null/tmpdir",
work_directory=Path("/dev/null/tmpdir"),
spectra_indices=None,
read_file=mock_read_file,
write_files=mock_write_files,
Expand Down Expand Up @@ -129,17 +129,17 @@ def test_map_chunked_external_to_files(
passed_operation = mock_map_chunked_to_files.mock_calls[0].kwargs["operation"]
passed_operation(mock_reader, [3, 4, 5], mock_write_files)
mock_operation.assert_called_once_with(
"/dev/null/tmpdir/tmp_file.imzML",
Path("/dev/null/tmpdir/tmp_file.imzML"),
["test1.imzML", "test2.imzML"],
)
mock_imzml_write_file.assert_called_once_with(
"/dev/null/tmpdir/tmp_file.imzML", imzml_mode=mock_reader.imzml_mode
Path("/dev/null/tmpdir/tmp_file.imzML"), imzml_mode=mock_reader.imzml_mode
)

def test_get_split_modes_and_paths_when_spectra_indices_none(self) -> None:
self.mock_config.get_splits_count.return_value = 3
split_modes_and_paths = self.mock_parallel._get_split_modes_and_paths(
work_directory="/dev/null/mock",
work_directory=Path("/dev/null/mock"),
read_file=MagicMock(name="mock_read_file", n_spectra=20),
write_files=[
MagicMock(name="mock_write_file_1", imzml_mode=ImzmlModeEnum.CONTINUOUS),
Expand All @@ -152,9 +152,9 @@ def test_get_split_modes_and_paths_when_spectra_indices_none(self) -> None:
(
ImzmlModeEnum.CONTINUOUS,
[
"/dev/null/mock/chunk_f0_t0.imzML",
"/dev/null/mock/chunk_f0_t1.imzML",
"/dev/null/mock/chunk_f0_t2.imzML",
Path("/dev/null/mock/chunk_f0_t0.imzML"),
Path("/dev/null/mock/chunk_f0_t1.imzML"),
Path("/dev/null/mock/chunk_f0_t2.imzML"),
],
),
split_modes_and_paths[0],
Expand All @@ -163,9 +163,9 @@ def test_get_split_modes_and_paths_when_spectra_indices_none(self) -> None:
(
ImzmlModeEnum.PROCESSED,
[
"/dev/null/mock/chunk_f1_t0.imzML",
"/dev/null/mock/chunk_f1_t1.imzML",
"/dev/null/mock/chunk_f1_t2.imzML",
Path("/dev/null/mock/chunk_f1_t0.imzML"),
Path("/dev/null/mock/chunk_f1_t1.imzML"),
Path("/dev/null/mock/chunk_f1_t2.imzML"),
],
),
split_modes_and_paths[1],
Expand All @@ -175,7 +175,7 @@ def test_get_split_modes_and_paths_when_spectra_indices_none(self) -> None:
def test_get_split_modes_and_paths_when_spectra_indices_present(self) -> None:
self.mock_config.get_splits_count.return_value = 3
split_modes_and_paths = self.mock_parallel._get_split_modes_and_paths(
work_directory="/dev/null/mock",
work_directory=Path("/dev/null/mock"),
read_file=MagicMock(name="mock_read_file", n_spectra=20),
write_files=[
MagicMock(name="mock_write_file_1", imzml_mode=ImzmlModeEnum.CONTINUOUS),
Expand All @@ -188,9 +188,9 @@ def test_get_split_modes_and_paths_when_spectra_indices_present(self) -> None:
(
ImzmlModeEnum.CONTINUOUS,
[
"/dev/null/mock/chunk_f0_t0.imzML",
"/dev/null/mock/chunk_f0_t1.imzML",
"/dev/null/mock/chunk_f0_t2.imzML",
Path("/dev/null/mock/chunk_f0_t0.imzML"),
Path("/dev/null/mock/chunk_f0_t1.imzML"),
Path("/dev/null/mock/chunk_f0_t2.imzML"),
],
),
split_modes_and_paths[0],
Expand All @@ -199,9 +199,9 @@ def test_get_split_modes_and_paths_when_spectra_indices_present(self) -> None:
(
ImzmlModeEnum.PROCESSED,
[
"/dev/null/mock/chunk_f1_t0.imzML",
"/dev/null/mock/chunk_f1_t1.imzML",
"/dev/null/mock/chunk_f1_t2.imzML",
Path("/dev/null/mock/chunk_f1_t0.imzML"),
Path("/dev/null/mock/chunk_f1_t1.imzML"),
Path("/dev/null/mock/chunk_f1_t2.imzML"),
],
),
split_modes_and_paths[1],
Expand Down Expand Up @@ -287,17 +287,17 @@ def test_merge_results(self, mock_read_file, method_merge) -> None:
[
call(
input_files=[
{"read_file": "/dev/null/mock/continuous_0.imzML"},
{"read_file": "/dev/null/mock/continuous_1.imzML"},
{"read_file": "/dev/null/mock/continuous_2.imzML"},
{"read_file": Path("/dev/null/mock/continuous_0.imzML")},
{"read_file": Path("/dev/null/mock/continuous_1.imzML")},
{"read_file": Path("/dev/null/mock/continuous_2.imzML")},
],
output_file=mock_write_file_0,
),
call(
input_files=[
{"read_file": "/dev/null/mock/processed_0.imzML"},
{"read_file": "/dev/null/mock/processed_1.imzML"},
{"read_file": "/dev/null/mock/processed_2.imzML"},
{"read_file": Path("/dev/null/mock/processed_0.imzML")},
{"read_file": Path("/dev/null/mock/processed_1.imzML")},
{"read_file": Path("/dev/null/mock/processed_2.imzML")},
],
output_file=mock_write_file_1,
),
Expand Down

0 comments on commit 3c150e2

Please sign in to comment.