Merging trk (#242)

* added new files

* add files & fixes

* fix

* activ sigmoid

* infer for lungs

* tests

* models uploaded, code readability

* hemor_contrast old recover 1

* hemor_contrast old recover 2

* revert old test changes

* refactor rtk to new files

* restore old coco files

* restore old test file

* rename experiments

* lung description check

* fix tests

* fix tests

* tests removed

* float comparison in tests

* remove selfwritten iou

* get test back

* black

* an attempt to pass tests

* an attempt to pass tests

* test back

* Fixed gdcm error

* Fixed gdcm error

* fix path bug

* more tests

* fixes and more tests

* argparse tests

* tests more

* remove redund comments

---------

Co-authored-by: Ainur Karimov <[email protected]>
Co-authored-by: InnopolisU <[email protected]>

config/datasets/classification/lung_description_decision_datamodule.yaml

@@ -8,13 +8,15 @@ markup_info: Набор данных содержит описание легк
 date_time: 13.09.2023
 
 _target_: innofw.core.datamodules.pandas_datamodules.lung_description_decision_datamodule.LungDescriptionDecisionPandasDataModule
-target_col: decision
+target_col: "decision"
 
 train:
-  source: ./data/ainur/infer/labels.csv
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/rtk/labels.zip
+  target: ./data/lung_description/train
 test:
-  source: ./data/ainur/infer/labels.csv
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/rtk/labels.zip
+  target: ./data/lung_description/test
 infer:
-  source: ./data/ainur/infer/labels.csv
-
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/rtk/labels.zip
+  target: ./data/lung_description/infer
 

config/datasets/semantic-segmentation/segmentation_brain_complex.yaml

@@ -0,0 +1,17 @@
+_target_: innofw.core.datamodules.lightning_datamodules.coco_rtk.DicomCocoComplexingDataModule
+date_time: 30.09.2022
+description: stroke segmentation dataset
+markup_info: segmentation masks
+name: stroke segmentation
+task:
+  - image-segmentation
+
+test:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/stroke/test.zip
+  target: ./data/stroke/test
+train:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/stroke/train.zip
+  target: ./data/stroke/train
+infer:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/rtk/complex_infer.zip
+  target: ./data/complex/infer

config/datasets/semantic-segmentation/segmentation_brain_rtk.yaml

@@ -0,0 +1,17 @@
+_target_: innofw.core.datamodules.lightning_datamodules.coco_rtk.DicomCocoDataModuleRTK
+date_time: 30.09.2022
+description: stroke segmentation dataset
+markup_info: segmentation masks
+name: stroke segmentation
+task:
+  - image-segmentation
+
+test:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/stroke/test.zip
+  target: ./data/rtk/test
+train:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/stroke/train.zip
+  target: ./data/rtk/train
+infer:
+  source: https://api.blackhole.ai.innopolis.university/public-datasets/rtk/infer.zip
+  target: ./data/rtk/infer

config/experiments/semantic-segmentation/AK_081224_Yjc97FX_unet_brain_complex.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+defaults:
+  - override /models: semantic-segmentation/unet_brain
+  - override /datasets: semantic-segmentation/segmentation_brain_complex.yaml
+  - override /augmentations_train: none #segmentation_stroke.yaml
+  - override /augmentations_val: none
+  - override /augmentations_test: none
+  - override /losses: segmentation_losses.yaml
+models:
+  in_channels: 1
+project: "segmentation"
+task: "image-segmentation"
+random_seed: 42
+stop_param: 10
+epochs: 300
+weights_freq: 1
+batch_size: 10

config/experiments/semantic-segmentation/AK_081224_gwVOeQ_unet_brain_rtk.yaml

@@ -0,0 +1,17 @@
+# @package _global_
+defaults:
+  - override /models: semantic-segmentation/unet_brain
+  - override /datasets: semantic-segmentation/segmentation_brain_rtk.yaml
+  - override /augmentations_train: none #segmentation_stroke.yaml
+  - override /augmentations_val: none
+  - override /augmentations_test: none
+  - override /losses: segmentation_losses.yaml
+models:
+  in_channels: 1
+project: "segmentation"
+task: "image-segmentation"
+random_seed: 42
+stop_param: 10
+epochs: 300
+weights_freq: 1
+batch_size: 10

config/models/classification/lung_description_decision.yaml

@@ -6,8 +6,8 @@ steps:
   - - "vectorizer"
     - _target_: sklearn.feature_extraction.text.TfidfVectorizer
       max_df: 1000
-      stop_words:
-        - russian
+#      stop_words:
+#        - russian
       min_df: 10
   - - "classifier"
     - _target_: sklearn.linear_model.LogisticRegression

config/models/semantic-segmentation/unet_brain.yaml

@@ -0,0 +1,8 @@
+_target_: segmentation_models_pytorch.FPN
+description: Base Unet segmentation model with 1 channel input
+name: FPN
+in_channels: 1
+classes: 4
+encoder_name: 'efficientnet-b7'
+encoder_weights:
+activation: sigmoid

examples/infer_contrasting_ct_dicom_brain.sh

@@ -1 +1,9 @@
-python3 ./innofw/utils/data_utils/preprocessing/CT_hemorrhage_contrast.py
+data_path=$1
+out_path=$2
+
+if [ -z "$data_path" ]; then
+	data_path="https://api.blackhole.ai.innopolis.university/public-datasets/rtk/infer.zip"
+	echo "Using default data path $data_path"
+fi
+
+python innofw/utils/data_utils/preprocessing/CT_hemorrhage_contrast_rtk.py  --input "$data_path" --output "$out_path"

examples/infer_ct_brain_hemorrhage_detection.sh

@@ -1 +1,13 @@
-python infer.py experiments=semantic-segmentation/SK_180822_qmciwj41_unet_brain 'ckpt_path=https://api.blackhole.ai.innopolis.university/pretrained/segmentation_unet_brain.pt'
+data_path=$1
+experiments="semantic-segmentation/AK_081224_gwVOeQ_unet_brain_rtk"
+ckpt_path="https://api.blackhole.ai.innopolis.university/pretrained/segmentation_rtk_brain.pt"
+if [ -z "$data_path" ]
+then
+  python infer.py experiments=$experiments \
+    "ckpt_path=$ckpt_path"
+else
+  python infer.py experiments=$experiments \
+    "ckpt_path=$ckpt_path" \
+    "++datasets.infer.target='$data_path'" \
+    "++datasets.infer.source='$data_path'"
+fi

examples/infer_ct_brain_hemorrhage_segmentation.sh

@@ -1 +1,14 @@
-python infer.py experiments=semantic-segmentation/SK_180822_qmciwj41_unet_brain 'ckpt_path=https://api.blackhole.ai.innopolis.university/pretrained/segmentation_unet_brain.pt'
+data_path=$1
+experiments="semantic-segmentation/AK_081224_gwVOeQ_unet_brain_rtk"
+ckpt_path="https://api.blackhole.ai.innopolis.university/pretrained/segmentation_rtk_brain.pt"
+
+if [ -z "$data_path" ]
+then
+  python infer.py experiments=$experiments \
+    "ckpt_path=$ckpt_path"
+else
+  python infer.py experiments=$experiments \
+    "ckpt_path=$ckpt_path" \
+    "++datasets.infer.target='$data_path'" \
+    "++datasets.infer.source='$data_path'"
+fi

examples/infer_ct_mri_complexing.sh

@@ -1 +1,13 @@
-python infer.py experiments=semantic-segmentation/SK_100923_uner_brain_mri.yaml 'ckpt_path=https://api.blackhole.ai.innopolis.university/pretrained/segmentation_unet_brain_complex.pt'
+data_path=$1
+ckpt_path="https://api.blackhole.ai.innopolis.university/pretrained/segmentation_rtk_brain.pt"
+experiments="semantic-segmentation/AK_081224_Yjc97FX_unet_brain_complex.yaml"
+
+if [ -z "$data_path" ]
+then
+  python infer.py experiments=$experiments \
+    "ckpt_path=$ckpt_path"
+else
+  python infer.py experiments=$experiments \
+    "++datasets.infer.source='$data_path'" \
+    "ckpt_path=$ckpt_path"
+fi

examples/infer_lung_description_decision.sh

@@ -1,6 +1,6 @@
 export HYDRA_FULL_ERROR=1
 export experiments="classification/AK_130923_fbFMFDe1_lung_description_decision.yaml"
-export model_weights="/home/ainur/Desktop/innopolis/text/pipe.pkl"
+export model_weights="https://api.blackhole.ai.innopolis.university/pretrained/lungs_description.pkl"
 export data_source=$1
 if [ -z "$data_source" ]
 then

examples/metrics_contrasting_ct_dicom_brain.sh

@@ -0,0 +1,7 @@
+output=$1
+if [ -z "$output" ]; then
+	output="../innofw/logs/infer/contrast"
+	output+="/$(ls $output -tr | tail -n 1)"
+	output+="/$(ls $output -tr | tail -n 1)"
+fi
+python innofw/utils/data_utils/preprocessing/CT_hemorrhage_contrast_metrics.py -o "$output"

examples/metrics_ct_brain_hemorrhage_detection.sh

@@ -0,0 +1,14 @@
+data_path=$1
+out=$2
+if [ -z "$data_path" ]; then
+	data_path="../innofw/data/rtk/infer/"
+	echo "Using default data path $data_path"
+fi
+
+if [ -z "$output" ]; then
+	output="../innofw/logs/infer/segmentation/semantic-segmentation/AK_081224_gwVOeQ_unet_brain_rtk/"
+	output+="$(ls $output | tail -n 1)"
+	echo "Using default output path $output"
+fi
+
+python innofw/utils/data_utils/rtk/CT_hemorrhage_metrics.py -i "$data_path" -o "$output" -t "detection"

examples/metrics_ct_brain_hemorrhage_segmentation.sh

@@ -0,0 +1,14 @@
+data_path=$1
+output=$2
+if [ -z "$data_path" ]; then
+	data_path="../innofw/data/rtk/infer/"
+	echo "Using default data path $data_path"
+fi
+
+if [ -z "$output" ]; then
+	output="../innofw/logs/infer/segmentation/semantic-segmentation/AK_081224_gwVOeQ_unet_brain_rtk/"
+	output+="$(ls $output -tr| tail -n 1)"
+	echo "Using default output path $output"
+fi
+
+python innofw/utils/data_utils/rtk/CT_hemorrhage_metrics.py -i "$data_path" -o "$output" -t "segmentation"

examples/metrics_ct_mri_complexing.sh

@@ -0,0 +1,13 @@
+data_path=$1
+output=$2
+
+if [ -z "$data_path" ]; then
+	data_path="../innofw/data/complex/infer"
+	echo "Using default data path $data_path"
+fi
+
+if [ -z "$output" ]; then
+	output="../innofw/logs/infer/segmentation/semantic-segmentation/AK_081224_Yjc97FX_unet_brain_complex.yaml/"
+	output+="$(ls $output -tr | tail -n 1)"
+fi
+python innofw/utils/data_utils/rtk/CT_complexing_metrics.py -i $data_path -o $output

examples/metrics_lung_description_decision.sh

@@ -0,0 +1,14 @@
+data_path=$1
+if [ -z "$data_path" ]; then
+	data_path="./data/lung_description/infer/labels.csv"
+	echo "Using default data path $data_path"
+fi
+
+if [ -z "$output" ]; then
+	output="./logs/infer/lung_description_decision/classification/AK_130923_fbFMFDe1_lung_description_decision.yaml/"
+	output+="$(ls $output -tr| tail -n 1)/"
+	output+="$(ls $output -tr| tail -n 1)"
+	echo "Using default output path $output"
+fi
+
+python innofw/utils/data_utils/rtk/lungs_description_metrics.py -i "$data_path" -o "$output"