version 1.9.0 closes #9 #24

README.md

@@ -1,15 +1,10 @@
 # IBL Spike Sorting
 
-This is a Python port of the original MATLAB version of [Kilosort 2.5](https://github.com/MouseLand/Kilosort), written by Marius Pachitariu.
-
-TODO: changes
-The modifications are described in [this presentation](https://docs.google.com/presentation/d/18bD_vQU45bLDSxd_QW2kbzEg8_5o3dqO5Gm9huXbNKE/edit?usp=sharing)
-We are working on an updated version of the whitepaper, but in the meantime, you can refer to [the previous version here](https://doi.org/10.6084/m9.figshare.19705522.v3).
-
+This is the implementation of the IBL spike sorting pipeline described on this white paper: (https://doi.org/10.6084/m9.figshare.19705522.v4).
+The clustering part is based on the original MATLAB version of [Kilosort 2.5](https://github.com/MouseLand/Kilosort), written by Marius Pachitariu.
 
 ## Usage
 
-
 We provide a few datasets to explore parametrization and test on several brain regions.
 The smallest dataset is a 100 seconds excerpt to test the installation. Here is the minimal working example:
 
@@ -50,8 +45,8 @@ if __name__ == "__main__":
 ### System Requirements
 
 The code makes extensive use of the GPU via the CUDA framework. A high-end NVIDIA GPU with at least 8GB of memory is required.
-The solution has been deployed and tested on Cuda 12+ and Python 3.11.
-In June 2024, pyfftw was still not compatible with Python 3.12.
+The solution has been deployed and tested on Cuda 12+ and Python 3.12 in October 2024.
+
 
 ### Python environment
 
@@ -70,9 +65,9 @@ Installation for cuda 11.8
     pip install cupy-cuda11x
     pip install -e .
 
-Installation for cuda 12.1
+Installation for the latest cuda:
 
-    pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu12.1
+    pip3 install torch torchvision torchaudio
     pip install cupy-cuda12x
     pip install -e .
 
@@ -85,8 +80,4 @@ from iblsorter.utils import cuda_installation_test
 cuda_installation_test()
 ```
 
-Then we can run the integration test
-
-```shell
-python ./examples/integration_100s.py /mnt/s1/spikesorting/integration_tests/stand-alone/
-```
+Then we can run the integration test.

iblsorter/__init__.py

@@ -12,4 +12,4 @@
 from .main import run, run_export, run_spikesort, run_preprocess
 from .io.probes import np1_probe, np2_probe, np2_4shank_probe
 
-__version__ = '1.9.0a'
+__version__ = '1.9.0'

iblsorter/params.py

@@ -1,10 +1,12 @@
 from math import ceil
+from pathlib import Path
 from typing import List, Optional, Tuple, Literal
 import yaml
 
 import numpy as np
 from pydantic import BaseModel, Field, field_validator, DirectoryPath
 
+import iblsorter
 from .utils import Bunch
 
 
@@ -14,7 +16,9 @@ class IntegrationConfig(BaseModel):
     delete: bool
 
 
-def load_integration_config(yaml_path) -> IntegrationConfig:
+def load_integration_config(yaml_path=None) -> IntegrationConfig:
+    if yaml_path is None:
+        yaml_path = Path(iblsorter.__file__).parents[1].joinpath('integration', 'config.yaml')
     with open(yaml_path, 'r') as fid:
         config = yaml.safe_load(fid)
     return IntegrationConfig.model_validate(config)

integration/ibl_spikesorting_task.py

@@ -5,32 +5,39 @@
 import logging
 import shutil
 from pathlib import Path
-from ibllib.pipes.ephys_tasks import SpikeSorting
-from iblsorter.ibl import download_test_data
 
-logger = logging.getLogger('iblsorter')
+from iblutil.util import setup_logger
+from ibllib.pipes.ephys_tasks import SpikeSorting
 
-SCRATCH_FOLDER = Path('/home/olivier/scratch')
-PATH_INTEGRATION = Path("/mnt/s1/spikesorting/integration_tests")
+from viz import reports
+import iblsorter
+from iblsorter.params import load_integration_config
 
+logger = setup_logger('iblsorter', level='DEBUG')
+config = load_integration_config()
+override_params = {}
 
 if __name__ == "__main__":
-
-    PATH_INTEGRATION.joinpath(f"{datetime.datetime.now().isoformat().replace(':', '')}.temp").touch()
-    download_test_data(PATH_INTEGRATION.joinpath('ibl'))
-    path_probe = PATH_INTEGRATION.joinpath("ibl", "probe01")
-    testing_path = PATH_INTEGRATION.joinpath("testing_output")
-    shutil.rmtree(testing_path, ignore_errors=True)
-    pname = path_probe.parts[-1]
-    session_path = testing_path.joinpath('iblsort', 'Subjects', 'iblsort_subject', '2024-07-16', '001')
-    raw_ephys_data_path = session_path.joinpath('raw_ephys_data', pname)
-    logger.info('copying raw_ephys_data to session path')
+    path_probe = config.integration_data_path.joinpath("testing_input", "ibl_spikesorting_task")
+    output_dir = config.integration_data_path.joinpath(
+        'testing_output', 'ibl_spikesorting_task', f"{iblsorter.__version__}")
+    shutil.rmtree(output_dir, ignore_errors=True)
+    session_path = output_dir.joinpath('Subjects', 'algernon', '2024-07-16', '001')
+    raw_ephys_data_path = session_path.joinpath('raw_ephys_data', 'probe01')
+    logger.info('copying raw_ephys_data to session path - removing existing folder if it exists')
     shutil.rmtree(raw_ephys_data_path, ignore_errors=True)
     raw_ephys_data_path.mkdir(parents=True, exist_ok=True)
     shutil.copytree(path_probe, raw_ephys_data_path, dirs_exist_ok=True)
-    logger.info(f"iblsort run for probe {pname} in session {session_path}")
-    ssjob = SpikeSorting(session_path, one=None, pname=pname, device_collection='raw_ephys_data', location="local", scratch_folder=SCRATCH_FOLDER)
+    logger.info(f"iblsort run for probe01 in session {session_path}")
+    ssjob = SpikeSorting(session_path, one=None, pname='probe01', device_collection='raw_ephys_data', location="local", scratch_folder=config.scratch_dir)
     ssjob.run()
     assert ssjob.status == 0
     ssjob.assert_expected_outputs()
-    logger.info("Test is complete and outputs validated - exiting now")
+    logger.info("Outputs are validated. Compute report")
+    alf_path = session_path.joinpath('alf', 'probe01', 'iblsorter')
+    reports.qc_plots_metrics(bin_file=next(raw_ephys_data_path.glob('*.ap.cbin')), pykilosort_path=alf_path, out_path=output_dir, raster_plot=True,
+                             raw_plots=True, summary_stats=False, raster_start=0., raster_len=100., raw_start=50., raw_len=0.15,
+                             vmax=0.05, d_bin=5, t_bin=0.001)
+    logger.info("Remove raw data copy")
+    shutil.rmtree(raw_ephys_data_path, ignore_errors=True)
+    logger.info(f"Exiting now, test data results in {output_dir}")

integration/integration_100s.py

@@ -7,10 +7,8 @@
 from iblsorter.params import load_integration_config
 from viz import reports
 
-file_config = Path(iblsorter.__file__).parents[1].joinpath('integration', 'config.yaml')
-
 setup_logger('iblsorter', level='DEBUG')
-config = load_integration_config(file_config)
+config = load_integration_config()
 override_params = {}
 label = ""
 
@@ -24,14 +22,12 @@ def run_integration_test(bin_file):
 
     :param bin_file:
     """
-    output_dir = config.integration_data_path.joinpath(f"{iblsorter.__version__}" + label, bin_file.name.split('.')[0])
-    ks_output_dir = output_dir.joinpath('pykilosort')
+    output_dir = config.integration_data_path.joinpath(
+        'testing_output', 'integration_100s', f"{iblsorter.__version__}" + label, bin_file.name.split('.')[0])
+    ks_output_dir = output_dir.joinpath('iblsorter')
     alf_path = ks_output_dir.joinpath('alf')
-
-    # this can't be outside of a function, otherwise each multiprocessing job will execute this code!
     shutil.rmtree(config.scratch_dir, ignore_errors=True)
     config.scratch_dir.mkdir(exist_ok=True)
-
     ks_output_dir.mkdir(parents=True, exist_ok=True)
 
     params = ibl_pykilosort_params(bin_file)
@@ -48,10 +44,10 @@ def run_integration_test(bin_file):
         intermediate_directory.mkdir(exist_ok=True)
         shutil.copy(pre_proc_file, intermediate_directory)
 
-    reports.qc_plots_metrics(bin_file=bin_file, pykilosort_path=alf_path, raster_plot=True, raw_plots=True, summary_stats=False,
-                             raster_start=0., raster_len=100., raw_start=50., raw_len=0.15,
+    reports.qc_plots_metrics(bin_file=bin_file, pykilosort_path=alf_path, out_path=output_dir, raster_plot=True,
+                             raw_plots=True, summary_stats=False, raster_start=0., raster_len=100., raw_start=50., raw_len=0.15,
                              vmax=0.05, d_bin=5, t_bin=0.001)
 
 
 if __name__ == "__main__":
-    run_integration_test(config.integration_data_path.joinpath("imec_385_100s.ap.bin"))
+    run_integration_test(config.integration_data_path.joinpath('testing_input', 'integration_100s', 'imec_385_100s.ap.bin'))

requirements.txt

@@ -7,7 +7,7 @@ matplotlib
 mock
 numba
 numpy
-ibllib @ git+https://github.com/int-brain-lab/ibllib@iblsort
+ibllib >= 2.40.0
 ibl-neuropixel >= 1.5.0
 ipython
 phylib >= 2.5.0