From 499e5ceb3d976defbc3f49bc5a5009f7797d80d5 Mon Sep 17 00:00:00 2001 From: Peter Sobolewski Date: Thu, 11 Aug 2022 23:15:50 -0400 Subject: [PATCH 1/6] thread worker progress for run_on_current --- src/napari_serialcellpose/serial_widget.py | 38 +++++++++++++--------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/napari_serialcellpose/serial_widget.py b/src/napari_serialcellpose/serial_widget.py index 42f3ff1..376ea53 100644 --- a/src/napari_serialcellpose/serial_widget.py +++ b/src/napari_serialcellpose/serial_widget.py @@ -4,6 +4,7 @@ from qtpy.QtCore import Qt import magicgui.widgets from napari.layers import Image +from napari.qt import create_worker, thread_worker from .folder_list_widget import FolderList from .serial_analysis import run_cellpose, load_props, load_allprops @@ -312,22 +313,27 @@ def _on_click_run_on_current(self): reg_props = [k for k in self.check_props.keys() if self.check_props[k].isChecked()] # run cellpose - segmented = run_cellpose( - image_path=image_path, - cellpose_model=self.cellpose_model, - output_path=self.output_folder, - diameter=diameter, - flow_threshold=self.flow_threshold.value(), - cellprob_threshold=self.cellprob_threshold.value(), - clear_border=self.check_clear_border.isChecked(), - channel_to_segment=channel_to_segment, - channel_helper=channel_helper, - channel_measure=channel_analysis, - properties=reg_props, - options_file=self.options_file_path, - force_no_rgb=self.check_no_rgb.isChecked(), - ) - self.viewer.add_labels(segmented, name='mask') + seg_worker = create_worker(run_cellpose, + image_path=image_path, + cellpose_model=self.cellpose_model, + output_path=self.output_folder, + diameter=diameter, + flow_threshold=self.flow_threshold.value(), + cellprob_threshold=self.cellprob_threshold.value(), + clear_border=self.check_clear_border.isChecked(), + channel_to_segment=channel_to_segment, + channel_helper=channel_helper, + channel_measure=channel_analysis, + properties=reg_props, + options_file=self.options_file_path, + force_no_rgb=self.check_no_rgb.isChecked(), + _progress=True + ) + def get_seg_worker(labels): + self.viewer.add_labels(labels, name='mask') + + seg_worker.start() + seg_worker.returned.connect(get_seg_worker) if self.output_folder is not None: props = load_props(self.output_folder, image_path) self.add_table_props(props) From 5316bbc204c8cf20e387433d972be8e67b743c96 Mon Sep 17 00:00:00 2001 From: Peter Sobolewski Date: Thu, 11 Aug 2022 23:22:05 -0400 Subject: [PATCH 2/6] thread worker progres for run on folder --- src/napari_serialcellpose/serial_widget.py | 44 +++++++++++++--------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/src/napari_serialcellpose/serial_widget.py b/src/napari_serialcellpose/serial_widget.py index 376ea53..c21febe 100644 --- a/src/napari_serialcellpose/serial_widget.py +++ b/src/napari_serialcellpose/serial_widget.py @@ -5,6 +5,8 @@ import magicgui.widgets from napari.layers import Image from napari.qt import create_worker, thread_worker +from napari.utils.notifications import show_info + from .folder_list_widget import FolderList from .serial_analysis import run_cellpose, load_props, load_allprops @@ -331,7 +333,8 @@ def _on_click_run_on_current(self): ) def get_seg_worker(labels): self.viewer.add_labels(labels, name='mask') - + + show_info('Running Segmentation...') seg_worker.start() seg_worker.returned.connect(get_seg_worker) if self.output_folder is not None: @@ -356,23 +359,28 @@ def _on_click_run_on_folder(self): channel_to_segment, channel_helper, channel_analysis = self.get_channels_to_use() reg_props = [k for k in self.check_props.keys() if self.check_props[k].isChecked()] - for batch in file_list_partition: - run_cellpose( - image_path=batch, - cellpose_model=self.cellpose_model, - output_path=self.output_folder, - diameter=diameter, - flow_threshold=self.flow_threshold.value(), - cellprob_threshold=self.cellprob_threshold.value(), - clear_border=self.check_clear_border.isChecked(), - channel_to_segment=channel_to_segment, - channel_helper=channel_helper, - channel_measure=channel_analysis, - properties=reg_props, - options_file=self.options_file_path, - force_no_rgb=self.check_no_rgb.isChecked(), - ) - + @thread_worker(progress={'total': len(file_list_partition), 'desc': 'Running batch segmentation'}) + def run_batch(file_list_partition): + for batch in file_list_partition: + run_cellpose( + image_path=batch, + cellpose_model=self.cellpose_model, + output_path=self.output_folder, + diameter=diameter, + flow_threshold=self.flow_threshold.value(), + cellprob_threshold=self.cellprob_threshold.value(), + clear_border=self.check_clear_border.isChecked(), + channel_to_segment=channel_to_segment, + channel_helper=channel_helper, + channel_measure=channel_analysis, + properties=reg_props, + options_file=self.options_file_path, + force_no_rgb=self.check_no_rgb.isChecked(), + ) + + show_info('Running Segmentation...') + batch_worker = run_batch(file_list_partition) + batch_worker.start() self._on_click_load_summary() def get_channels_to_use(self): From 3577cbc292a2300157aeb0fc64a0c7e1fa92132c Mon Sep 17 00:00:00 2001 From: Peter Sobolewski Date: Thu, 11 Aug 2022 23:23:49 -0400 Subject: [PATCH 3/6] Make sure to yield in for loop --- src/napari_serialcellpose/serial_widget.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/napari_serialcellpose/serial_widget.py b/src/napari_serialcellpose/serial_widget.py index c21febe..4ff99cd 100644 --- a/src/napari_serialcellpose/serial_widget.py +++ b/src/napari_serialcellpose/serial_widget.py @@ -362,7 +362,7 @@ def _on_click_run_on_folder(self): @thread_worker(progress={'total': len(file_list_partition), 'desc': 'Running batch segmentation'}) def run_batch(file_list_partition): for batch in file_list_partition: - run_cellpose( + yield run_cellpose( image_path=batch, cellpose_model=self.cellpose_model, output_path=self.output_folder, From 3ab187ad661de6ed075a5c8e730bfef5501db313 Mon Sep 17 00:00:00 2001 From: Guillaume Witz Date: Sat, 13 Aug 2022 16:02:58 +0200 Subject: [PATCH 4/6] fix for multiple outputs of run_cellpose --- src/napari_serialcellpose/serial_widget.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/napari_serialcellpose/serial_widget.py b/src/napari_serialcellpose/serial_widget.py index 516daf0..5f1e6c1 100644 --- a/src/napari_serialcellpose/serial_widget.py +++ b/src/napari_serialcellpose/serial_widget.py @@ -316,6 +316,8 @@ def _on_click_run_on_current(self): channel_analysis_names = [x.text() for x in self.qcbox_channel_analysis.selectedItems()] reg_props = [k for k in self.check_props.keys() if self.check_props[k].isChecked()] + self.viewer.layers.events.inserted.disconnect(self._on_change_layers) + # run cellpose seg_worker = create_worker(run_cellpose, image_path=image_path, @@ -334,15 +336,16 @@ def _on_click_run_on_current(self): force_no_rgb=self.check_no_rgb.isChecked(), _progress=True ) - def get_seg_worker(labels): - self.viewer.add_labels(labels, name='mask') + def get_seg_worker(output): + self.viewer.add_labels(output[0], name='mask') + if len(reg_props) > 0: + self.add_table_props(output[1]) show_info('Running Segmentation...') seg_worker.start() seg_worker.returned.connect(get_seg_worker) - self.viewer.layers.events.inserted.disconnect(self._on_change_layers) - self.viewer.add_labels(segmented, name='mask') + self.viewer.layers.events.inserted.connect(self._on_change_layers) if len(reg_props) > 0: self.add_table_props(props) @@ -625,4 +628,4 @@ def __init__(self, parent=None, col=1, row=1, width=6, height=4, dpi=100): for j in range(col): self.ax[i,j] = fig.add_subplot(row, col, count) count+=1 - super(MplCanvas, self).__init__(fig) \ No newline at end of file + super(MplCanvas, self).__init__(fig) From 52bcbf8f66379ef27697eefd139befcbfa1bd4f0 Mon Sep 17 00:00:00 2001 From: Peter Sobolewski Date: Sun, 14 Aug 2022 22:51:09 -0400 Subject: [PATCH 5/6] fix tests? --- .../_tests/test_widget.py | 114 +++++++++++------- src/napari_serialcellpose/serial_widget.py | 10 +- 2 files changed, 71 insertions(+), 53 deletions(-) diff --git a/src/napari_serialcellpose/_tests/test_widget.py b/src/napari_serialcellpose/_tests/test_widget.py index fe9736c..cbbbad9 100644 --- a/src/napari_serialcellpose/_tests/test_widget.py +++ b/src/napari_serialcellpose/_tests/test_widget.py @@ -1,15 +1,18 @@ from napari_serialcellpose import SerialWidget import numpy as np import pandas as pd - +import pytest from pathlib import Path +import time +import os +import tempfile import shutil def test_load_single_image(make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) - + mypath = Path('src/napari_serialcellpose/_tests/data/single_file_singlechannel/') widget.file_list.update_from_path(mypath) @@ -17,11 +20,11 @@ def test_load_single_image(make_napari_viewer): widget.file_list.setCurrentRow(0) assert len(viewer.layers) == 1 -def test_analyse_single_image_no_save(make_napari_viewer): +def test_analyse_single_image_no_save(qtbot, make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) - + mypath = Path('src/napari_serialcellpose/_tests/data/single_file_singlechannel/') widget.file_list.update_from_path(mypath) @@ -38,24 +41,24 @@ def test_analyse_single_image_no_save(make_napari_viewer): # set diameter and run segmentation widget.spinbox_diameter.setValue(70) widget._on_click_run_on_current() - + # check that segmentatio has been added, named 'mask' and results in 33 objects - assert len(viewer.layers) == 2 + def check_layers(): + assert len(viewer.layers) == 2 + + qtbot.waitUntil(check_layers, timeout=30000) assert viewer.layers[1].name == 'mask' assert viewer.layers[1].data.max() == 33 -def test_analyse_single_image_save(make_napari_viewer): +def test_analyse_single_image_save(qtbot, make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/single_file_multichannel') - output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_single') - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(exist_ok=True) - + output_dir = Path(tempfile.mkdtemp()) + widget.file_list.update_from_path(mypath) widget.output_folder = output_dir widget.file_list.setCurrentRow(0) @@ -76,23 +79,26 @@ def test_analyse_single_image_save(make_napari_viewer): widget.check_props['size'].setChecked(True) widget.check_props['intensity'].setChecked(True) widget.qcbox_channel_analysis.setCurrentRow(1) - widget._on_click_run_on_current() - assert len(list(output_dir.glob('*mask.tif'))) == 1 + def check_outputs(): + assert len(list(output_dir.glob('*mask.tif'))) == 1 + + qtbot.waitUntil(check_outputs, timeout=30000) + assert len(list(output_dir.joinpath('tables').glob('*_props.csv'))) == 1 + shutil.rmtree(output_dir) -def test_analyse_multi_image(make_napari_viewer): +def test_analyse_multi_image(qtbot, make_napari_viewer): """Test analysis of multiple images in a folder. No properties are analyzed.""" viewer = make_napari_viewer() widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/multifile/') - output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_multiple') - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(exist_ok=True) + + output_dir = Path(tempfile.mkdtemp()) + widget.file_list.update_from_path(mypath) widget.output_folder = output_dir @@ -101,23 +107,22 @@ def test_analyse_multi_image(make_napari_viewer): widget.qcbox_model_choice.setCurrentIndex( [widget.qcbox_model_choice.itemText(i) for i in range(widget.qcbox_model_choice.count())].index('cyto2')) widget.spinbox_diameter.setValue(70) - widget._on_click_run_on_current() + widget._on_click_run_on_folder() - assert len(list(output_dir.glob('*mask.tif'))) == 1 + def check_output(): + assert len(list(output_dir.glob('*mask.tif'))) == 4 - widget._on_click_run_on_folder() - assert len(list(output_dir.glob('*mask.tif'))) == 4 + qtbot.waitUntil(check_output, timeout=30000) + shutil.rmtree(output_dir) -def test_analyse_multi_image_props(make_napari_viewer): +def test_analyse_multi_image_props(qtbot, make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/multifile/') - output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_multiple3') - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(exist_ok=True) + output_dir = Path(tempfile.mkdtemp()) + widget.file_list.update_from_path(mypath) widget.output_folder = output_dir @@ -133,27 +138,27 @@ def test_analyse_multi_image_props(make_napari_viewer): widget.qcbox_channel_analysis.setCurrentRow(1) widget._on_click_run_on_folder() - assert len(list(output_dir.glob('*mask.tif'))) == 4 - # check that the properties are correct + def check_outputs(): + assert len(list(output_dir.glob('*mask.tif'))) == 4 + + qtbot.waitUntil(check_outputs, timeout=30000) + # check that the properties are correct df = pd.read_csv(output_dir.joinpath( - 'tables', - Path(widget.file_list.currentItem().text()).stem + '_props.csv' + 'tables', + Path(widget.file_list.currentItem().text()).stem + '_props.csv') ) - ) - # check number of columns in df - assert df.shape[1] == 8 + # check number of columns in df + assert df.shape[1] == 8 + -def test_analyse_multichannels(make_napari_viewer): +def test_analyse_multichannels(qtbot, make_napari_viewer): """Test that multiple channels can be used for intensity measurements""" viewer = make_napari_viewer() widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/single_file_multichannel/') - output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_single_multichannelprops') - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(exist_ok=True) + output_dir = Path(tempfile.mkdtemp()) widget.file_list.update_from_path(mypath) widget.output_folder = output_dir @@ -171,6 +176,11 @@ def test_analyse_multichannels(make_napari_viewer): widget._on_click_run_on_folder() + def check_outputs(): + assert len(list(output_dir.glob('*mask.tif'))) == 1 + + qtbot.waitUntil(check_outputs, timeout=30000) + # check that the properties are correct df = pd.read_csv(output_dir.joinpath( 'tables', @@ -180,16 +190,17 @@ def test_analyse_multichannels(make_napari_viewer): # check number of columns in df assert df.shape[1] == 11 -def test_mask_loading(make_napari_viewer): +def test_mask_loading(qtbot, make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/multifile/') output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_multiple2') - if output_dir.exists(): - shutil.rmtree(output_dir) - output_dir.mkdir(exist_ok=True) + # causes tests to hang, but here we don't need this check, we can check layers + #if output_dir.exists(): + # shutil.rmtree(output_dir) + #output_dir.mkdir(exist_ok=True) widget.file_list.update_from_path(mypath) widget.output_folder = output_dir @@ -200,6 +211,12 @@ def test_mask_loading(make_napari_viewer): widget.spinbox_diameter.setValue(70) widget._on_click_run_on_current() + # check that segmentation has been added + def check_layers(): + assert len(viewer.layers) == 3 + + qtbot.waitUntil(check_layers, timeout=30000) + # check that when selecting the second file, we get only 2 channels and no mask widget.file_list.setCurrentRow(1) assert len(viewer.layers) == 2 @@ -208,7 +225,7 @@ def test_mask_loading(make_napari_viewer): widget.file_list.setCurrentRow(0) assert len(viewer.layers) == 3 -def test_analyse_single_image_options_yml(make_napari_viewer): +def test_analyse_single_image_options_yml(qtbot, make_napari_viewer): viewer = make_napari_viewer() widget = SerialWidget(viewer) @@ -232,5 +249,10 @@ def test_analyse_single_image_options_yml(make_napari_viewer): widget._on_click_run_on_current() - # check that because of small diameter from yml file, we get only 5 elements + # check that segmentation has been added + def check_layers(): + assert len(viewer.layers) == 3 + + qtbot.waitUntil(check_layers, timeout=30000) + # check that because of small diameter from yml file, we get only 7 elements assert viewer.layers[2].data.max() == 7 \ No newline at end of file diff --git a/src/napari_serialcellpose/serial_widget.py b/src/napari_serialcellpose/serial_widget.py index 5f1e6c1..053b56a 100644 --- a/src/napari_serialcellpose/serial_widget.py +++ b/src/napari_serialcellpose/serial_widget.py @@ -336,21 +336,17 @@ def _on_click_run_on_current(self): force_no_rgb=self.check_no_rgb.isChecked(), _progress=True ) + def get_seg_worker(output): self.viewer.add_labels(output[0], name='mask') if len(reg_props) > 0: self.add_table_props(output[1]) + self.viewer.layers.events.inserted.connect(self._on_change_layers) show_info('Running Segmentation...') seg_worker.start() seg_worker.returned.connect(get_seg_worker) - - self.viewer.layers.events.inserted.connect(self._on_change_layers) - - if len(reg_props) > 0: - self.add_table_props(props) - self.viewer.layers.events.inserted.connect(self._on_change_layers) def _on_click_run_on_folder(self): """Run cellpose on all images in folder""" @@ -396,7 +392,7 @@ def run_batch(file_list_partition): batch_worker = run_batch(file_list_partition) batch_worker.start() - self._on_click_load_summary() + batch_worker.returned.connect(self._on_click_load_summary) def get_channels_to_use(self): """Translate selected channels in QCombox into indices. From 75f6ca78e6f75ea67b117cac00421e7248dad707 Mon Sep 17 00:00:00 2001 From: Peter Sobolewski Date: Sun, 14 Aug 2022 22:58:55 -0400 Subject: [PATCH 6/6] one more test fix --- src/napari_serialcellpose/_tests/test_widget.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/napari_serialcellpose/_tests/test_widget.py b/src/napari_serialcellpose/_tests/test_widget.py index cbbbad9..620eb80 100644 --- a/src/napari_serialcellpose/_tests/test_widget.py +++ b/src/napari_serialcellpose/_tests/test_widget.py @@ -196,11 +196,8 @@ def test_mask_loading(qtbot, make_napari_viewer): widget = SerialWidget(viewer) mypath = Path('src/napari_serialcellpose/_tests/data/multifile/') - output_dir = Path('src/napari_serialcellpose/_tests/data/analyzed_multiple2') - # causes tests to hang, but here we don't need this check, we can check layers - #if output_dir.exists(): - # shutil.rmtree(output_dir) - #output_dir.mkdir(exist_ok=True) + output_dir = Path(tempfile.mkdtemp()) + widget.file_list.update_from_path(mypath) widget.output_folder = output_dir