From f3ec3b7ce8ddc5c31ea58b54ac26e5fc1a136fc7 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Fri, 9 Jul 2021 17:37:45 +0100 Subject: [PATCH 01/20] Migrate to ONE2 --- iblvideo/motion_energy.py | 2 +- iblvideo/run.py | 9 +++++---- prototyping/3d/IBL_3d.py | 5 +++-- prototyping/motion_energy_pipeline.py | 15 ++++++++------- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/iblvideo/motion_energy.py b/iblvideo/motion_energy.py index 908d18e..51da072 100644 --- a/iblvideo/motion_energy.py +++ b/iblvideo/motion_energy.py @@ -14,7 +14,7 @@ import cv2 import logging -from oneibl.one import ONE +from one.api import ONE from ibllib.io.video import get_video_frames_preload, label_from_path from ibllib.io.extractors.camera import get_video_length from oneibl.stream import VideoStreamer diff --git a/iblvideo/run.py b/iblvideo/run.py index be717a9..8f2d558 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -12,7 +12,7 @@ import numpy as np -from oneibl.one import ONE +from one.api import ONE from oneibl.patcher import FTPPatcher from iblvideo.choiceworld import dlc from iblvideo.motion_energy import motion_energy @@ -189,7 +189,7 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No try: # Create ONE instance if none is given one = one or ONE() - session_path = one.path_from_eid(session_id) + session_path = one.eid2path(session_id) tdict = one.alyx.rest('tasks', 'list', django=f"name__icontains,DLC,session__id,{session_id}")[0] except IndexError: @@ -245,8 +245,9 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No # Download camera times and then force qc to use local data as dlc might not have # been updated on FlatIron at this stage try: - one.load(session_id, dataset_types=['camera.times'], download_only=True) - alf_path = one.path_from_eid(session_id).joinpath('alf') + datasets = one.datasets_from_type(session_id, 'camera.times') + one.load_datasets(session_id, datasets=datasets, download_only=True) + alf_path = one.eid2path(session_id).joinpath('alf') for cam in cams: # Only run if dlc actually exists if alf_path.joinpath(f'_ibl_{cam}Camera.dlc.pqt').exists(): diff --git a/prototyping/3d/IBL_3d.py b/prototyping/3d/IBL_3d.py index 84d520a..1995661 100644 --- a/prototyping/3d/IBL_3d.py +++ b/prototyping/3d/IBL_3d.py @@ -17,7 +17,7 @@ from scipy.spatial.transform import Rotation as R import alf.io -from oneibl.one import ONE +from one.api import ONE from pathlib import Path import mpl_toolkits.mplot3d.axes3d as p3 @@ -81,7 +81,8 @@ def GetXYs(eid, video_type, trial_range): assert all([i in a for i in dataset_types] ), 'For this eid, not all data available' - D = one.load(eid, dataset_types=dataset_types, dclass_output=True) + datasets = one.datasets_from_type(eid, dataset_types) + D = one.load_datasets(eid, datasets=datasets) alf_path = Path(D.local_path[0]).parent.parent / 'alf' video_data = alf_path.parent / 'raw_video_data' diff --git a/prototyping/motion_energy_pipeline.py b/prototyping/motion_energy_pipeline.py index 9fd43f2..16d0cb7 100644 --- a/prototyping/motion_energy_pipeline.py +++ b/prototyping/motion_energy_pipeline.py @@ -2,7 +2,7 @@ import numpy as np import subprocess import alf.io -from oneibl.one import ONE +from one.api import ONE from pathlib import Path import os import time @@ -35,10 +35,10 @@ def get_dlc_XYs(eid, video_type): a = one.list(eid,'dataset-types') if not all([x['dataset_type'] for x in a]): print('not all data available') - return - - - one.load(eid, dataset_types = dataset_types) + return + + datasets = one.datasets_from_type(eid, dataset_types) + one.load_datasets(eid, datasets=datasets) local_path = one.path_from_eid(eid) alf_path = local_path / 'alf' @@ -255,8 +255,9 @@ def load_compute_ROI_ME(eid): print('not all data available') return - # download all three raw videos - one.load(eid, dataset_types) + # download all three raw videos + datasets = one.datasets_from_type(eid, dataset_types) + one.load_datasets(eid, datasets=datasets) video_data = one.path_from_eid(eid) / 'raw_video_data' for video_type in ['body','left','right']: From ec06bac757adbbb15d2036b76436d83fa27d016f Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 13 Jul 2021 13:43:47 +0100 Subject: [PATCH 02/20] Rename datasets_from_type to type2datasets --- iblvideo/run.py | 2 +- prototyping/3d/IBL_3d.py | 2 +- prototyping/motion_energy_pipeline.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 8f2d558..a740706 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -245,7 +245,7 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No # Download camera times and then force qc to use local data as dlc might not have # been updated on FlatIron at this stage try: - datasets = one.datasets_from_type(session_id, 'camera.times') + datasets = one.type2datasets(session_id, 'camera.times') one.load_datasets(session_id, datasets=datasets, download_only=True) alf_path = one.eid2path(session_id).joinpath('alf') for cam in cams: diff --git a/prototyping/3d/IBL_3d.py b/prototyping/3d/IBL_3d.py index 1995661..30d06df 100644 --- a/prototyping/3d/IBL_3d.py +++ b/prototyping/3d/IBL_3d.py @@ -81,7 +81,7 @@ def GetXYs(eid, video_type, trial_range): assert all([i in a for i in dataset_types] ), 'For this eid, not all data available' - datasets = one.datasets_from_type(eid, dataset_types) + datasets = one.type2datasets(eid, dataset_types) D = one.load_datasets(eid, datasets=datasets) alf_path = Path(D.local_path[0]).parent.parent / 'alf' diff --git a/prototyping/motion_energy_pipeline.py b/prototyping/motion_energy_pipeline.py index 16d0cb7..e9e81e2 100644 --- a/prototyping/motion_energy_pipeline.py +++ b/prototyping/motion_energy_pipeline.py @@ -37,7 +37,7 @@ def get_dlc_XYs(eid, video_type): print('not all data available') return - datasets = one.datasets_from_type(eid, dataset_types) + datasets = one.type2datasets(eid, dataset_types) one.load_datasets(eid, datasets=datasets) local_path = one.path_from_eid(eid) alf_path = local_path / 'alf' @@ -256,7 +256,7 @@ def load_compute_ROI_ME(eid): return # download all three raw videos - datasets = one.datasets_from_type(eid, dataset_types) + datasets = one.type2datasets(eid, dataset_types) one.load_datasets(eid, datasets=datasets) video_data = one.path_from_eid(eid) / 'raw_video_data' From f713ff3d20d8ce0ded11c896257b38ca183e3c02 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 20 Jul 2021 10:07:46 +0100 Subject: [PATCH 03/20] Fix error import --- iblvideo/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index a740706..3878896 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -13,6 +13,7 @@ import numpy as np from one.api import ONE +from one.alf.exceptions import ALFObjectNotFound from oneibl.patcher import FTPPatcher from iblvideo.choiceworld import dlc from iblvideo.motion_energy import motion_energy @@ -22,10 +23,10 @@ from ibllib.pipes import tasks from ibllib.qc.dlc import DlcQC from ibllib.io.video import assert_valid_label -from ibllib.exceptions import ALFObjectNotFound _logger = logging.getLogger('ibllib') + # re-using the Task class allows to not re-write all the logging, error management # and automatic settings of task statuses in the database def _format_timer(timer): From 3eb0e9227effc8ac6f6d5197bc68fc7d71ca8c96 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 20 Jul 2021 10:18:08 +0100 Subject: [PATCH 04/20] No cache --- iblvideo/run.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 3878896..8971760 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -100,7 +100,7 @@ def _run(self, cams=('left', 'body', 'right'), version=__version__, frames=None, video_intact, clobber_vid, attempt = False, False, 0 while video_intact is False and attempt < 3: dset = self.one.alyx.rest('datasets', 'list', session=session_id, - name=f'_iblrig_{cam}Camera.raw.mp4') + name=f'_iblrig_{cam}Camera.raw.mp4', no_cache=True) file_mp4 = self.one.download_dataset(dset[0], clobber=clobber_vid) # Check if video is downloaded completely, otherwise retry twice video_intact = self._video_intact(file_mp4) @@ -192,7 +192,8 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No one = one or ONE() session_path = one.eid2path(session_id) tdict = one.alyx.rest('tasks', 'list', - django=f"name__icontains,DLC,session__id,{session_id}")[0] + django=f"name__icontains,DLC,session__id,{session_id}", + no_cache=True)[0] except IndexError: print(f"No DLC task found for session {session_id}") return -1 @@ -204,7 +205,8 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No vids = [dset['name'] for dset in one.alyx.rest('datasets', 'list', django=(f'session__id,{session_id},' 'data_format__name,mp4,' - f'name__icontains,camera') + f'name__icontains,camera'), + no_cache=True )] no_vid = [cam for cam in cams if f'_iblrig_{cam}Camera.raw.mp4' not in vids] if len(no_vid) > 0: @@ -212,7 +214,8 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No log_str = '\n'.join([f"No raw video file found for {no_cam}Camera." for no_cam in no_vid]) patch_data = {'log': log_str, 'status': 'Errored'} - one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data) + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, + no_cache=True) return -1 else: # set a flag in local session folder to later resume if interrupted @@ -223,7 +226,8 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No status = task.run(cams=cams, version=version, frames=frames, overwrite=overwrite) patch_data = {'time_elapsed_secs': task.time_elapsed_secs, 'log': task.log, 'status': 'Errored' if status == -1 else 'Complete'} - one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data) + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, + no_cache=True) # register the data using the FTP patcher if task.outputs: # it is safer to instantiate the FTP right before transfer to prevent time-out @@ -260,16 +264,19 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No # If the camera.times don't exist we cannot run QC, but the DLC task shouldn't fail # Make sure to not overwrite the task log if that has already been updated tdict = one.alyx.rest('tasks', 'list', - django=f"name__icontains,DLC,session__id,{session_id}")[0] + django=f"name__icontains,DLC,session__id,{session_id}", + no_cache=True)[0] patch_data = {'log': tdict['log'] + '\n\n' + traceback.format_exc()} - one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data) + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, + no_cache=True) except BaseException: # Make sure to not overwrite the task log if that has already been updated tdict = one.alyx.rest('tasks', 'list', - django=f"name__icontains,DLC,session__id,{session_id}")[0] + django=f"name__icontains,DLC,session__id,{session_id}", + no_cache=True)[0] patch_data = {'log': tdict['log'] + '\n\n' + traceback.format_exc(), 'status': 'Errored'} - one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data) + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, no_cache=True) return -1 # Remove in progress flag session_path.joinpath('dlc_started').unlink() @@ -317,7 +324,7 @@ def run_queue(machine=None, target_versions=(__version__), delta = (datetime.now() - last_query).total_seconds() if (delta > delta_query) or (count == 0): last_query = datetime.now() - all_tasks = one.alyx.rest('tasks', 'list', name='EphysDLC') + all_tasks = one.alyx.rest('tasks', 'list', name='EphysDLC', no_cache=True) task_queue = [t for t in all_tasks if t['status'] in statuses and (t['version'] is None or t['version'] not in target_versions)] task_queue = sorted(task_queue, key=lambda k: k['priority'], reverse=True) From 49be49d61e4ad6359d8bb614eafc4343e78a47ce Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 20 Jul 2021 12:16:30 +0100 Subject: [PATCH 05/20] Change in ONE2 --- iblvideo/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 8971760..380f98d 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -101,7 +101,7 @@ def _run(self, cams=('left', 'body', 'right'), version=__version__, frames=None, while video_intact is False and attempt < 3: dset = self.one.alyx.rest('datasets', 'list', session=session_id, name=f'_iblrig_{cam}Camera.raw.mp4', no_cache=True) - file_mp4 = self.one.download_dataset(dset[0], clobber=clobber_vid) + file_mp4 = self.one._download_dataset(dset[0], clobber=clobber_vid) # Check if video is downloaded completely, otherwise retry twice video_intact = self._video_intact(file_mp4) if video_intact is False: From dcc3b483e7369de95c0e68260aefcdba873044b0 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 21 Jul 2021 09:32:21 +0100 Subject: [PATCH 06/20] one._par to one.alyx._par --- iblvideo/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 380f98d..71e4f0d 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -71,7 +71,7 @@ def _video_intact(file_mp4): def _run(self, cams=('left', 'body', 'right'), version=__version__, frames=None, overwrite=False): - session_id = self.one.eid_from_path(self.session_path) + session_id = self.one.path2eid(self.session_path) timer = OrderedDict() dlc_results, me_results, me_rois = [], [], [] @@ -303,7 +303,7 @@ def run_queue(machine=None, target_versions=(__version__), one = ONE() # Loop until n_sessions is reached or something breaks - machine = machine or one._par.ALYX_LOGIN + machine = machine or one.alyx._par.ALYX_LOGIN status_dict = {} # Check if any interrupted local sessions are present From 2320ab75b0c13b9fcb013ff652911ef02b553b88 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 21 Jul 2021 15:24:43 +0100 Subject: [PATCH 07/20] Try to catch 403 error in patcher --- iblvideo/run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 71e4f0d..ed9adf9 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -5,10 +5,14 @@ import time import cv2 import warnings +import shutil +import requests + from packaging.version import parse from glob import glob from datetime import datetime from collections import OrderedDict +from pathlib import Path import numpy as np @@ -303,7 +307,7 @@ def run_queue(machine=None, target_versions=(__version__), one = ONE() # Loop until n_sessions is reached or something breaks - machine = machine or one.alyx._par.ALYX_LOGIN + machine = machine or one.alyx.user status_dict = {} # Check if any interrupted local sessions are present From aabef0831a4a20545b3ab4c4d9420c58eb6c4d89 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Thu, 22 Jul 2021 09:18:03 +0100 Subject: [PATCH 08/20] fix ftp patcher import and revisions --- iblvideo/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index ed9adf9..a0eab66 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -18,7 +18,7 @@ from one.api import ONE from one.alf.exceptions import ALFObjectNotFound -from oneibl.patcher import FTPPatcher +from ibllib.oneibl.patcher import FTPPatcher from iblvideo.choiceworld import dlc from iblvideo.motion_energy import motion_energy from iblvideo.weights import download_weights From 2f390ef2013b69b765a068852015544068e71a21 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Fri, 1 Oct 2021 15:54:34 +0100 Subject: [PATCH 09/20] working on tf2 instructions --- README.md | 53 ++++++++++++++----------------------------------- iblvideo/run.py | 5 +---- 2 files changed, 16 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 1c7a7a8..560fb89 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,10 @@ output = dlc("Path/to/file.mp4") from iblvideo import run_session run_session("db156b70-8ef8-4479-a519-ba6f8c4a73ee") ``` -### Running 10 sessions from the queue +### Running the queue ```python from iblvideo import run_queue -run_queue(n_sessions=10) +run_queue(machine='mymachine') ``` ## Accessing results @@ -40,59 +40,36 @@ DLC results are stored on the Flatrion server, with the `dataset_type` being `ca Install local server as per [this instruction](https://docs.google.com/document/d/1NYVlVD8OkwRYUaPeHo3ZFPuwpv_E5zgUVjLsV0V5Ko4). -Install CUDA 10.0 libraries as documented [here](https://docs.google.com/document/d/1UyXUOx21mwrpBtCcS51avnikmyCPCzXEtTRaTetH-Mo) to match the TensorFlow version 1.13.1 required for DLC. +Install CUDA 11.2 libraries as documented [here](https://docs.google.com/document/d/1UyXUOx21mwrpBtCcS51avnikmyCPCzXEtTRaTetH-Mo/edit#heading=h.39mk45fhbn1l) Install cuDNN, an extension of the Cuda Toolkit for deep neural networks: Download cuDNN from FlatIron as shown below, or find it online. ```bash -wget --user iblmember --password check_your_one_settings http://ibl.flatironinstitute.org/resources/cudnn-10.0-linux-x64-v7.6.5.32.tgz -tar -xvf cudnn-10.0-linux-x64-v7.6.5.32.tgz -sudo cp cuda/include/cudnn.h /usr/local/cuda-10.0/include -sudo cp cuda/lib64/libcudnn* /usr/local/cuda-10.0/lib64 -sudo chmod a+r /usr/local/cuda-10.0/include/cudnn.h /usr/local/cuda-10.0/lib64/libcudnn* +wget --user iblmember --password check_your_one_settings http://ibl.flatironinstitute.org/resources/cudnn-11.2-linux-x64-v8.1.1.33.tgz +tar -xvf cudnn-11.2-linux-x64-v8.1.1.33.tgz +sudo cp cuda/include/cudnn.h /usr/local/cuda-11.2/include +sudo cp cuda/lib64/libcudnn* /usr/local/cuda-11.2/lib64 +sudo chmod a+r /usr/local/cuda-11.2/include/cudnn.h /usr/local/cuda-11.2/lib64/libcudnn* ``` -(optional): check CUDNN installation or for troubleshooting only) -Download and unzip https://ibl.flatironinstitute.org/resources/cudnn_samples_v7.zip -If necessary, setup your CUDA environment variables with the version you want to test - -``` -cd cudnn_samples_v7/mnistCUDNN/ -make clean && make -./mnistCUDNN -``` - -Should print a message that finishes with Test passed ! - ### Create a Python environment with TensorFlow and DLC -Install a few things system wide and then python3.7 - -```bash -sudo apt update -sudo apt install software-properties-common -sudo add-apt-repository ppa:deadsnakes/ppa -sudo apt-get install python3.7-tk -sudo apt install python3.7 python3.7-dev -``` - -Create and activate a Python 3.7 environment called e.g. dlcenv +Create and activate an environment called e.g. dlcenv ```bash mkdir -p ~/Documents/PYTHON/envs cd ~/Documents/PYTHON/envs -virtualenv dlcenv --python=python3.7 +virtualenv dlcenv source ~/Documents/PYTHON/envs/dlcenv/bin/activate ``` -Install packages (please observe order of those commands and Ubuntu version, you may need to change the wxpython link!) +Install packages ```bash pip install -U setuptools -pip install git+https://github.com/int-brain-lab/ibllib.git@1.11.0 -pip install https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04/wxPython-4.0.7-cp37-cp37m-linux_x86_64.whl -pip install tensorflow-gpu==1.13.1 -pip install deeplabcut==2.1.10 +pip install git+https://github.com/int-brain-lab/ibllib.git +pip install tensorflow +pip install deeplabcut ``` ### Test if tensorflow and deeplabcut installation was successful @@ -100,7 +77,7 @@ pip install deeplabcut==2.1.10 Export environment variable to avoid tensorflow issue and point to CUDA libraries ```bash export TF_FORCE_GPU_ALLOW_GROWTH='true' -export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:/lib/nccl/cuda-10:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:/usr/local/cuda-11.2/extras/CUPTI/lib64:/lib/nccl/cuda-11:$LD_LIBRARY_PATH ``` Try importing tensorflow and deeplabcut diff --git a/iblvideo/run.py b/iblvideo/run.py index a0eab66..abf4a73 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -6,13 +6,10 @@ import cv2 import warnings import shutil -import requests -from packaging.version import parse from glob import glob from datetime import datetime from collections import OrderedDict -from pathlib import Path import numpy as np @@ -312,7 +309,7 @@ def run_queue(machine=None, target_versions=(__version__), # Check if any interrupted local sessions are present if restart_local is True: - local_tmp = glob(one._par.CACHE_DIR + '/*lab*/Subjects/*/*/*/dlc_started') + local_tmp = glob(one.alyx._par.CACHE_DIR + '/*lab*/Subjects/*/*/*/dlc_started') if len(local_tmp) > 0: local_sessions = list(set([one.eid_from_path(local) for local in local_tmp])) n_sessions -= len(local_sessions) From cca11dee5f09ff453a6a32a643ce2e2927f67a44 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Fri, 1 Oct 2021 18:36:12 +0200 Subject: [PATCH 10/20] Install and tests working locally --- README.md | 20 ++++---------------- iblvideo/motion_energy.py | 2 -- iblvideo/tests/download_test_data.py | 4 ++-- iblvideo/weights.py | 4 ++-- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 560fb89..465f510 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Create and activate an environment called e.g. dlcenv ```bash mkdir -p ~/Documents/PYTHON/envs cd ~/Documents/PYTHON/envs -virtualenv dlcenv +virtualenv dlcenv --python=python3.8 source ~/Documents/PYTHON/envs/dlcenv/bin/activate ``` @@ -74,31 +74,19 @@ pip install deeplabcut ### Test if tensorflow and deeplabcut installation was successful -Export environment variable to avoid tensorflow issue and point to CUDA libraries -```bash -export TF_FORCE_GPU_ALLOW_GROWTH='true' -export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:/usr/local/cuda-11.2/extras/CUPTI/lib64:/lib/nccl/cuda-11:$LD_LIBRARY_PATH -``` - -Try importing tensorflow and deeplabcut ``` python -c 'import deeplabcut, tensorflow' ``` -(If you get a pandas incompatibility error, try this: -``` -pip uninstall pandas -pip install pandas==1.1.5 -``` -) +If you get errors make sure the cuda libraries are correctly added to your paths in .bashrc as explained [here](https://docs.google.com/document/d/1UyXUOx21mwrpBtCcS51avnikmyCPCzXEtTRaTetH-Mo/edit#heading=h.ryxfckh2bbpl). -If this is successful (no errors) you can set up an alias in your .bashrc file to easily enter the iblvideo environment: +Once the import goes through without errors (it is ok to get the warning that you cannot use the GUI), you can set up an alias in your .bashrc file to easily enter the iblvideo environment: ```bash nano ~/.bashrc ``` Enter this line under the other aliases: ```bash -alias dlcenv="export TF_FORCE_GPU_ALLOW_GROWTH='true'; export LD_LIBRARY_PATH=/usr/local/cuda-10.0/lib64:/usr/local/cuda-10.0/extras/CUPTI/lib64:$LD_LIBRARY_PATH; source ~/Documents/PYTHON/envs/dlcenv/bin/activate" +alias dlcenv="source ~/Documents/PYTHON/envs/dlcenv/bin/activate" ``` After opening a new terminal you should be able to type `dlcenv` and end up in an environment in which you can import tensorflow and deeplabcut like above. diff --git a/iblvideo/motion_energy.py b/iblvideo/motion_energy.py index 51da072..94f41b8 100644 --- a/iblvideo/motion_energy.py +++ b/iblvideo/motion_energy.py @@ -14,10 +14,8 @@ import cv2 import logging -from one.api import ONE from ibllib.io.video import get_video_frames_preload, label_from_path from ibllib.io.extractors.camera import get_video_length -from oneibl.stream import VideoStreamer _log = logging.getLogger('ibllib') diff --git a/iblvideo/tests/download_test_data.py b/iblvideo/tests/download_test_data.py index 4ecdc2a..1acd1a3 100644 --- a/iblvideo/tests/download_test_data.py +++ b/iblvideo/tests/download_test_data.py @@ -1,8 +1,8 @@ import logging import shutil from pathlib import Path -from ibllib.io import params -from oneibl.webclient import http_download_file +from iblutil.io import params +from one.webclient import http_download_file from iblvideo import __version__ _logger = logging.getLogger('ibllib') diff --git a/iblvideo/weights.py b/iblvideo/weights.py index e5a7443..aecaa85 100644 --- a/iblvideo/weights.py +++ b/iblvideo/weights.py @@ -2,8 +2,8 @@ import logging import shutil from pathlib import Path -from ibllib.io import params -from oneibl.webclient import http_download_file +from iblutil.io import params +from one.webclient import http_download_file from iblvideo import __version__ _logger = logging.getLogger('ibllib') From 148ae4a277ce8b0f8b03551645b02048e82bd7d0 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Fri, 1 Oct 2021 19:08:25 +0200 Subject: [PATCH 11/20] Update Readme --- README.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 465f510..6779e30 100644 --- a/README.md +++ b/README.md @@ -125,10 +125,11 @@ The version of DLC weights and DLC test data are synchronized with the MAJOR.MIN If you update any of the DLC weights, you also need to update the MINOR version of the code and the DLC test data, and vice versa. 1. For the weights, create a new directory called `weights_v{MAJOR}.{MINOR}` and copy the new weights, plus any unchanged weights into it. -2. Make a new `dlc_test_data_v{MAJOR}.{MINOR}` directory, with subdirectories `input` and `output`. +2. Make a new `dlc_test_data_v{MAJOR}.{MINOR}` directory. 3. Copy the three videos from the `input` folder of the previous version dlc_test_data to the new one. -4. Create the three parquet files to go in `output` by running iblvideo.dlc() with the new weights folder as `path_dlc`, and each of the videos in the new `input` folder as `file_mp4`. -5. Zip and upload the new weights and test data folders to FlatIron : +4. Create the three parquet files to go in by running iblvideo.dlc() with the new weights folder as `path_dlc`, and each of the videos in the new `input` folder as `file_mp4`. +5. Rename the newly created folder `alf` inside the dlc_test_data folder into `output`. +6. Zip and upload the new weights and test data folders to FlatIron : ``` /resources/dlc/weights_v{MAJOR}.{MINOR}.zip /integration/dlc/test_data/dlc_test_data_v{MAJOR}.{MINOR}.zip From 94573452927a75e71226e43154244ac4ea88bdc8 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Fri, 1 Oct 2021 19:31:31 +0200 Subject: [PATCH 12/20] Bump version --- iblvideo/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iblvideo/__init__.py b/iblvideo/__init__.py index 9d852f4..e228b25 100644 --- a/iblvideo/__init__.py +++ b/iblvideo/__init__.py @@ -1,5 +1,5 @@ -__version__ = '1.2.0' # This is the only place where the version is hard coded, only adapt here +__version__ = '2.0.0' # This is the only place where the version is hard coded, only adapt here import deeplabcut From 503ae91ad69a5177ca19a0f697797349e82e7112 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 5 Oct 2021 10:54:03 +0200 Subject: [PATCH 13/20] Update dockers --- docker/Dockerfile | 27 --------------------------- docker/Dockerfile.base | 17 +++++++---------- docker/Dockerfile.v2.0 | 11 +++++++++++ docker/README_DOCKER.md | 4 ++-- docker/docker-compose.yaml | 4 ++-- 5 files changed, 22 insertions(+), 41 deletions(-) delete mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile.v2.0 diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index fd7db97..0000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -#docker build -t ibl/dlc:base . -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 - -# link the cuda libraries -ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH - -# setup time zone for tz -ENV TZ=Europe/Paris -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Install python3.7 -RUN apt-get update -RUN apt-get install -y software-properties-common -RUN add-apt-repository ppa:deadsnakes/ppa -RUN apt-get update -RUN apt-get install -y python3.7 python3.7-dev python3.7-tk python3-pip python3.7-venv git ffmpeg libgtk-3-dev - -# Install Python dependencies -ARG PYTHON=python3.7 -ENV LANG C.UTF-8 -RUN ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 -RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install "dask[complete]" -RUN python3 -m pip install ibllib -RUN python3 -m pip install https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04/wxPython-4.0.7-cp37-cp37m-linux_x86_64.whl -RUN python3 -m pip install tensorflow-gpu==1.13.1 -RUN python3 -m pip install deeplabcut diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 11540ff..426f376 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -1,5 +1,5 @@ #docker build -t internationalbrainlab/dlc:base -f Dockerfile.base -FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 +FROM nvidia/cuda:11.2.2-cudnn8-devel-ubuntu18.04 # link the cuda libraries ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH @@ -8,22 +8,19 @@ ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH ENV TZ=Europe/Paris RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone -# Install python3.7 +# Install python3.8 and some other packages RUN apt-get update RUN apt-get install -y software-properties-common RUN add-apt-repository ppa:deadsnakes/ppa RUN apt-get update -RUN apt-get install -y python3.7 python3.7-dev python3.7-tk python3-pip python3.7-venv git ffmpeg libgtk-3-dev +RUN apt-get install -y python3.8 python3.8-dev python3.8-tk python3-pip python3.8-venv git ffmpeg libgtk-3-dev # Install Python dependencies -ARG PYTHON=python3.7 +ARG PYTHON=python3.8 ENV LANG C.UTF-8 RUN ln -sf /usr/bin/${PYTHON} /usr/local/bin/python3 RUN python3 -m pip install --upgrade pip -RUN python3 -m pip install "dask[complete]" -RUN python3 -m pip install ibllib>=1.9.0 -RUN python3 -m pip install https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-18.04/wxPython-4.0.7-cp37-cp37m-linux_x86_64.whl -RUN python3 -m pip install tensorflow-gpu==1.13.1 +RUN python3 -m pip install -U setuptools +RUN python3 -m pip install git+https://github.com/int-brain-lab/ibllib.git +RUN python3 -m pip install tensorflow RUN python3 -m pip install deeplabcut - -ENV TF_FORCE_GPU_ALLOW_GROWTH 'true' diff --git a/docker/Dockerfile.v2.0 b/docker/Dockerfile.v2.0 new file mode 100644 index 0000000..230252c --- /dev/null +++ b/docker/Dockerfile.v2.0 @@ -0,0 +1,11 @@ +#docker build -t internationalbrainlab/dlc:v2.0 -f Dockerfile.v2.0 --no-cache . +FROM internationalbrainlab/dlc:base + +# copy the weights and test data +COPY weights_v2.0 weights_v2.0 +COPY me_test_data me_test_data +COPY dlc_test_data_v2.0 dlc_test_data_v2.0 + +# clone the code +RUN git clone --depth 1 --single-branch --branch one2 https://github.com/int-brain-lab/iblvideo.git +ENV PYTHONPATH=/iblvideo diff --git a/docker/README_DOCKER.md b/docker/README_DOCKER.md index d29578d..add4137 100644 --- a/docker/README_DOCKER.md +++ b/docker/README_DOCKER.md @@ -104,7 +104,7 @@ Update the image version in `docker-compose.yaml` test and queue configurations. cd ~/Documents/PYTHON git clone https://github.com/int-brain-lab/iblvideo cd iblvideo/docker -docker build -t internationalbrainlab/dlc:base -f Dockerfile.base # this one will take a long time +docker build -t internationalbrainlab/dlc:base -f Dockerfile.base --no-cache . # this one will take a long time ``` Test the image by accessing a shell inside of the container: @@ -112,7 +112,7 @@ Test the image by accessing a shell inside of the container: ```shell docker run -it --rm --gpus all -u $(id -u):$(id -g) -v /mnt/s0/Data/FlatIron:/mnt/s0/Data/FlatIron -v ~/Documents/PYTHON/iblvideo:/root internationalbrainlab/dlc:base python3 -from import deeplabcut +import deeplabcut ``` Eventually send the image to dockerhub diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 259f9d0..f1ae67d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -17,14 +17,14 @@ services: - ~/Documents/PYTHON/iblvideo:/iblvideo - ~:/root queue: - image: internationalbrainlab/dlc:v1.2 + image: internationalbrainlab/dlc:v2.0 command: python3 -c "from iblvideo import run_queue; run_queue()" runtime: nvidia volumes: - ~:/root - /mnt/s0/Data/FlatIron:/mnt/s0/Data/FlatIron test: - image: internationalbrainlab/dlc:v1.2 + image: internationalbrainlab/dlc:v2.0 command: bash -c "nvcc --version; pytest -s /iblvideo/iblvideo/tests" runtime: nvidia volumes: From d6654564697bebf63cbf2dcc395d1de84ab3d703 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 5 Oct 2021 11:02:08 +0200 Subject: [PATCH 14/20] Try to fix some issues with queue --- iblvideo/run.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index abf4a73..024abd4 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -278,14 +278,15 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No no_cache=True)[0] patch_data = {'log': tdict['log'] + '\n\n' + traceback.format_exc(), 'status': 'Errored'} one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, no_cache=True) + if session_path.joinpath('dlc_started').exists(): + session_path.joinpath('dlc_started').unlink() return -1 # Remove in progress flag session_path.joinpath('dlc_started').unlink() return status -def run_queue(machine=None, target_versions=(__version__), - statuses=('Empty', 'Complete', 'Errored', 'Waiting'), +def run_queue(machine=None, target_versions=(__version__), statuses=('Empty', 'Waiting', 'Complete'), restart_local=True, overwrite=True, n_sessions=1000, delta_query=600, **kwargs): """ From 77421ca7aab4ffa4f4994f74b730a5af19e607ca Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 5 Oct 2021 11:07:15 +0200 Subject: [PATCH 15/20] Some ONE2 updates --- iblvideo/run.py | 2 +- prototyping/motion_energy_pipeline.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index 024abd4..a9051b1 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -312,7 +312,7 @@ def run_queue(machine=None, target_versions=(__version__), statuses=('Empty', 'W if restart_local is True: local_tmp = glob(one.alyx._par.CACHE_DIR + '/*lab*/Subjects/*/*/*/dlc_started') if len(local_tmp) > 0: - local_sessions = list(set([one.eid_from_path(local) for local in local_tmp])) + local_sessions = list(set([one.path2eid(local) for local in local_tmp])) n_sessions -= len(local_sessions) for eid in local_sessions: print(f'Restarting local session {eid}') diff --git a/prototyping/motion_energy_pipeline.py b/prototyping/motion_energy_pipeline.py index e9e81e2..1086419 100644 --- a/prototyping/motion_energy_pipeline.py +++ b/prototyping/motion_energy_pipeline.py @@ -39,7 +39,7 @@ def get_dlc_XYs(eid, video_type): datasets = one.type2datasets(eid, dataset_types) one.load_datasets(eid, datasets=datasets) - local_path = one.path_from_eid(eid) + local_path = one.eid2path(eid) alf_path = local_path / 'alf' cam0 = alf.io.load_object( @@ -258,7 +258,7 @@ def load_compute_ROI_ME(eid): # download all three raw videos datasets = one.type2datasets(eid, dataset_types) one.load_datasets(eid, datasets=datasets) - video_data = one.path_from_eid(eid) / 'raw_video_data' + video_data = one.eid2path(eid) / 'raw_video_data' for video_type in ['body','left','right']: From 219bc659a617759d9a02162cb4335be6aeea9693 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 5 Oct 2021 11:10:05 +0200 Subject: [PATCH 16/20] Update target versions in docker-compose --- docker/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index f1ae67d..87441d7 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -18,7 +18,7 @@ services: - ~:/root queue: image: internationalbrainlab/dlc:v2.0 - command: python3 -c "from iblvideo import run_queue; run_queue()" + command: python3 -c "from iblvideo import run_queue; run_queue(target_versions=('1.2.0', '2.0.0'))" runtime: nvidia volumes: - ~:/root From cd7630d9608c75ada010c962d871fad3c500f370 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 6 Oct 2021 16:41:47 +0100 Subject: [PATCH 17/20] Include python 3.8 in install --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 6779e30..94a6c40 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,16 @@ sudo chmod a+r /usr/local/cuda-11.2/include/cudnn.h /usr/local/cuda-11.2/lib64/l ### Create a Python environment with TensorFlow and DLC +Install python3.8 (if necessary) + +```bash +sudo apt update +sudo apt install software-properties-common +sudo add-apt-repository ppa:deadsnakes/ppa +sudo apt-get install python3.8-tk +sudo apt install python3.8 python3.8-dev +``` + Create and activate an environment called e.g. dlcenv ```bash From 245ea11a7ff166ed4bc3abea0d29a732f6b5e698 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Wed, 6 Oct 2021 17:01:25 +0100 Subject: [PATCH 18/20] Fix some edge cases during run --- iblvideo/run.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index a9051b1..f560e9a 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -238,13 +238,11 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No if len(outputs) > 0: for output in outputs: ftp_patcher.create_dataset(path=output) + # Update the version only now and only if new outputs are uploaded + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data={'version': version}) else: _logger.warning("No new outputs computed.") - # Update the version only now and only if new outputs are uploaded - if status == 0 and len(outputs) > 0: - one.alyx.rest('tasks', 'partial_update', id=tdict['id'], - data={'version': version}) - if status == 0 and remove_videos is True: + if remove_videos is True: shutil.rmtree(session_path.joinpath('raw_video_data'), ignore_errors=True) # Run DLC QC @@ -278,11 +276,12 @@ def run_session(session_id, machine=None, cams=('left', 'body', 'right'), one=No no_cache=True)[0] patch_data = {'log': tdict['log'] + '\n\n' + traceback.format_exc(), 'status': 'Errored'} one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data=patch_data, no_cache=True) - if session_path.joinpath('dlc_started').exists(): - session_path.joinpath('dlc_started').unlink() return -1 # Remove in progress flag session_path.joinpath('dlc_started').unlink() + # Set status to Incomplete if the length of cameras was < 3 but everything passed + if status == 0 and len(cams) < 3: + one.alyx.rest('tasks', 'partial_update', id=tdict['id'], data={'status': 'Incomplete'}, no_cache=True) return status From 992bc8154f75d0079f6412adc6f90eae89fb3797 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Mon, 11 Oct 2021 10:41:57 +0100 Subject: [PATCH 19/20] Bring back allow growth env --- README.md | 10 +++++++++- docker/Dockerfile.base | 2 ++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94a6c40..7bd1156 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,13 @@ pip install tensorflow pip install deeplabcut ``` +Export environment variables +(the second command might already be set up with your cuda install) +```bash +export TF_FORCE_GPU_ALLOW_GROWTH='true' +export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:/usr/local/cuda-11.2/extras/CUPTI/lib64:$LD_LIBRARY_PATH +``` + ### Test if tensorflow and deeplabcut installation was successful ``` @@ -96,7 +103,8 @@ nano ~/.bashrc ``` Enter this line under the other aliases: ```bash -alias dlcenv="source ~/Documents/PYTHON/envs/dlcenv/bin/activate" +alias dlcenv="export TF_FORCE_GPU_ALLOW_GROWTH='true'; export LD_LIBRARY_PATH=/usr/local/cuda-11.2/lib64:/usr/local/cuda-11.2/extras/CUPTI/lib64:$LD_LIBRARY_PATH; source ~/Documents/PYTHON/envs/dlcenv/bin/activate" + ``` After opening a new terminal you should be able to type `dlcenv` and end up in an environment in which you can import tensorflow and deeplabcut like above. diff --git a/docker/Dockerfile.base b/docker/Dockerfile.base index 426f376..9a365b4 100644 --- a/docker/Dockerfile.base +++ b/docker/Dockerfile.base @@ -24,3 +24,5 @@ RUN python3 -m pip install -U setuptools RUN python3 -m pip install git+https://github.com/int-brain-lab/ibllib.git RUN python3 -m pip install tensorflow RUN python3 -m pip install deeplabcut + +ENV TF_FORCE_GPU_ALLOW_GROWTH 'true' \ No newline at end of file From adcd23dff4be05a861b4d9f2acc57100ebad93e8 Mon Sep 17 00:00:00 2001 From: juhuntenburg Date: Tue, 12 Oct 2021 13:35:56 +0100 Subject: [PATCH 20/20] Don't skip adding files when only motion energy fails --- iblvideo/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iblvideo/run.py b/iblvideo/run.py index f560e9a..085ca75 100644 --- a/iblvideo/run.py +++ b/iblvideo/run.py @@ -131,6 +131,7 @@ def _run(self, cams=('left', 'body', 'right'), version=__version__, frames=None, except BaseException: _logger.error(f'DLC {cam}Camera failed.\n' + traceback.format_exc()) self.status = -1 + # We dont' run motion energy, or add any files to be patched if dlc failed to run continue # If me outputs don't exist or should be overwritten, run me @@ -148,7 +149,6 @@ def _run(self, cams=('left', 'body', 'right'), version=__version__, frames=None, except BaseException: _logger.error(f'Motion energy {cam}Camera failed.\n' + traceback.format_exc()) self.status = -1 - continue if dlc_computed is True or dlc_mode == 'local': dlc_results.append(dlc_result)