From 9e053ca6ad759638de9b3a864e47cc2d67d731fb Mon Sep 17 00:00:00 2001 From: owinter Date: Mon, 28 Oct 2024 17:45:23 +0000 Subject: [PATCH] version 1.9.0 closes #9 #24 --- README.md | 23 +++++---------- iblsorter/__init__.py | 2 +- iblsorter/params.py | 6 +++- integration/ibl_spikesorting_task.py | 43 ++++++++++++++++------------ integration/integration_100s.py | 18 +++++------- requirements.txt | 2 +- 6 files changed, 46 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 1703525..17e4319 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,10 @@ # IBL Spike Sorting -This is a Python port of the original MATLAB version of [Kilosort 2.5](https://github.com/MouseLand/Kilosort), written by Marius Pachitariu. - -TODO: changes -The modifications are described in [this presentation](https://docs.google.com/presentation/d/18bD_vQU45bLDSxd_QW2kbzEg8_5o3dqO5Gm9huXbNKE/edit?usp=sharing) -We are working on an updated version of the whitepaper, but in the meantime, you can refer to [the previous version here](https://doi.org/10.6084/m9.figshare.19705522.v3). - +This is the implementation of the IBL spike sorting pipeline described on this white paper: (https://doi.org/10.6084/m9.figshare.19705522.v4). +The clustering part is based on the original MATLAB version of [Kilosort 2.5](https://github.com/MouseLand/Kilosort), written by Marius Pachitariu. ## Usage - We provide a few datasets to explore parametrization and test on several brain regions. The smallest dataset is a 100 seconds excerpt to test the installation. Here is the minimal working example: @@ -50,8 +45,8 @@ if __name__ == "__main__": ### System Requirements The code makes extensive use of the GPU via the CUDA framework. A high-end NVIDIA GPU with at least 8GB of memory is required. -The solution has been deployed and tested on Cuda 12+ and Python 3.11. -In June 2024, pyfftw was still not compatible with Python 3.12. +The solution has been deployed and tested on Cuda 12+ and Python 3.12 in October 2024. + ### Python environment @@ -70,9 +65,9 @@ Installation for cuda 11.8 pip install cupy-cuda11x pip install -e . -Installation for cuda 12.1 +Installation for the latest cuda: - pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu12.1 + pip3 install torch torchvision torchaudio pip install cupy-cuda12x pip install -e . @@ -85,8 +80,4 @@ from iblsorter.utils import cuda_installation_test cuda_installation_test() ``` -Then we can run the integration test - -```shell -python ./examples/integration_100s.py /mnt/s1/spikesorting/integration_tests/stand-alone/ -``` +Then we can run the integration test. diff --git a/iblsorter/__init__.py b/iblsorter/__init__.py index 6fd22ba..0fa4c0d 100755 --- a/iblsorter/__init__.py +++ b/iblsorter/__init__.py @@ -12,4 +12,4 @@ from .main import run, run_export, run_spikesort, run_preprocess from .io.probes import np1_probe, np2_probe, np2_4shank_probe -__version__ = '1.9.0a' +__version__ = '1.9.0' diff --git a/iblsorter/params.py b/iblsorter/params.py index fd0322f..2362a5b 100644 --- a/iblsorter/params.py +++ b/iblsorter/params.py @@ -1,10 +1,12 @@ from math import ceil +from pathlib import Path from typing import List, Optional, Tuple, Literal import yaml import numpy as np from pydantic import BaseModel, Field, field_validator, DirectoryPath +import iblsorter from .utils import Bunch @@ -14,7 +16,9 @@ class IntegrationConfig(BaseModel): delete: bool -def load_integration_config(yaml_path) -> IntegrationConfig: +def load_integration_config(yaml_path=None) -> IntegrationConfig: + if yaml_path is None: + yaml_path = Path(iblsorter.__file__).parents[1].joinpath('integration', 'config.yaml') with open(yaml_path, 'r') as fid: config = yaml.safe_load(fid) return IntegrationConfig.model_validate(config) diff --git a/integration/ibl_spikesorting_task.py b/integration/ibl_spikesorting_task.py index 9b2634f..af7498f 100644 --- a/integration/ibl_spikesorting_task.py +++ b/integration/ibl_spikesorting_task.py @@ -5,32 +5,39 @@ import logging import shutil from pathlib import Path -from ibllib.pipes.ephys_tasks import SpikeSorting -from iblsorter.ibl import download_test_data -logger = logging.getLogger('iblsorter') +from iblutil.util import setup_logger +from ibllib.pipes.ephys_tasks import SpikeSorting -SCRATCH_FOLDER = Path('/home/olivier/scratch') -PATH_INTEGRATION = Path("/mnt/s1/spikesorting/integration_tests") +from viz import reports +import iblsorter +from iblsorter.params import load_integration_config +logger = setup_logger('iblsorter', level='DEBUG') +config = load_integration_config() +override_params = {} if __name__ == "__main__": - - PATH_INTEGRATION.joinpath(f"{datetime.datetime.now().isoformat().replace(':', '')}.temp").touch() - download_test_data(PATH_INTEGRATION.joinpath('ibl')) - path_probe = PATH_INTEGRATION.joinpath("ibl", "probe01") - testing_path = PATH_INTEGRATION.joinpath("testing_output") - shutil.rmtree(testing_path, ignore_errors=True) - pname = path_probe.parts[-1] - session_path = testing_path.joinpath('iblsort', 'Subjects', 'iblsort_subject', '2024-07-16', '001') - raw_ephys_data_path = session_path.joinpath('raw_ephys_data', pname) - logger.info('copying raw_ephys_data to session path') + path_probe = config.integration_data_path.joinpath("testing_input", "ibl_spikesorting_task") + output_dir = config.integration_data_path.joinpath( + 'testing_output', 'ibl_spikesorting_task', f"{iblsorter.__version__}") + shutil.rmtree(output_dir, ignore_errors=True) + session_path = output_dir.joinpath('Subjects', 'algernon', '2024-07-16', '001') + raw_ephys_data_path = session_path.joinpath('raw_ephys_data', 'probe01') + logger.info('copying raw_ephys_data to session path - removing existing folder if it exists') shutil.rmtree(raw_ephys_data_path, ignore_errors=True) raw_ephys_data_path.mkdir(parents=True, exist_ok=True) shutil.copytree(path_probe, raw_ephys_data_path, dirs_exist_ok=True) - logger.info(f"iblsort run for probe {pname} in session {session_path}") - ssjob = SpikeSorting(session_path, one=None, pname=pname, device_collection='raw_ephys_data', location="local", scratch_folder=SCRATCH_FOLDER) + logger.info(f"iblsort run for probe01 in session {session_path}") + ssjob = SpikeSorting(session_path, one=None, pname='probe01', device_collection='raw_ephys_data', location="local", scratch_folder=config.scratch_dir) ssjob.run() assert ssjob.status == 0 ssjob.assert_expected_outputs() - logger.info("Test is complete and outputs validated - exiting now") + logger.info("Outputs are validated. Compute report") + alf_path = session_path.joinpath('alf', 'probe01', 'iblsorter') + reports.qc_plots_metrics(bin_file=next(raw_ephys_data_path.glob('*.ap.cbin')), pykilosort_path=alf_path, out_path=output_dir, raster_plot=True, + raw_plots=True, summary_stats=False, raster_start=0., raster_len=100., raw_start=50., raw_len=0.15, + vmax=0.05, d_bin=5, t_bin=0.001) + logger.info("Remove raw data copy") + shutil.rmtree(raw_ephys_data_path, ignore_errors=True) + logger.info(f"Exiting now, test data results in {output_dir}") diff --git a/integration/integration_100s.py b/integration/integration_100s.py index 3a7664a..0de1694 100644 --- a/integration/integration_100s.py +++ b/integration/integration_100s.py @@ -7,10 +7,8 @@ from iblsorter.params import load_integration_config from viz import reports -file_config = Path(iblsorter.__file__).parents[1].joinpath('integration', 'config.yaml') - setup_logger('iblsorter', level='DEBUG') -config = load_integration_config(file_config) +config = load_integration_config() override_params = {} label = "" @@ -24,14 +22,12 @@ def run_integration_test(bin_file): :param bin_file: """ - output_dir = config.integration_data_path.joinpath(f"{iblsorter.__version__}" + label, bin_file.name.split('.')[0]) - ks_output_dir = output_dir.joinpath('pykilosort') + output_dir = config.integration_data_path.joinpath( + 'testing_output', 'integration_100s', f"{iblsorter.__version__}" + label, bin_file.name.split('.')[0]) + ks_output_dir = output_dir.joinpath('iblsorter') alf_path = ks_output_dir.joinpath('alf') - - # this can't be outside of a function, otherwise each multiprocessing job will execute this code! shutil.rmtree(config.scratch_dir, ignore_errors=True) config.scratch_dir.mkdir(exist_ok=True) - ks_output_dir.mkdir(parents=True, exist_ok=True) params = ibl_pykilosort_params(bin_file) @@ -48,10 +44,10 @@ def run_integration_test(bin_file): intermediate_directory.mkdir(exist_ok=True) shutil.copy(pre_proc_file, intermediate_directory) - reports.qc_plots_metrics(bin_file=bin_file, pykilosort_path=alf_path, raster_plot=True, raw_plots=True, summary_stats=False, - raster_start=0., raster_len=100., raw_start=50., raw_len=0.15, + reports.qc_plots_metrics(bin_file=bin_file, pykilosort_path=alf_path, out_path=output_dir, raster_plot=True, + raw_plots=True, summary_stats=False, raster_start=0., raster_len=100., raw_start=50., raw_len=0.15, vmax=0.05, d_bin=5, t_bin=0.001) if __name__ == "__main__": - run_integration_test(config.integration_data_path.joinpath("imec_385_100s.ap.bin")) + run_integration_test(config.integration_data_path.joinpath('testing_input', 'integration_100s', 'imec_385_100s.ap.bin')) diff --git a/requirements.txt b/requirements.txt index d456dcb..d3e0bd8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,7 @@ matplotlib mock numba numpy -ibllib @ git+https://github.com/int-brain-lab/ibllib@iblsort +ibllib >= 2.40.0 ibl-neuropixel >= 1.5.0 ipython phylib >= 2.5.0