ultralytics adapter (#155)

* utils update

* .

* utils for different types of input

* Update utils.py

* Update utils.py

* Update concatenated_datamodule.py

* Update concatenated_datamodule.py

* fixed druprot

added list to tensor conversion

* Update segmentation.py

added image name, mask str conversion

* Update concatenated_datamodule.py

* changed pipeline and framework

* added datasets` config compatibility

* Delete EXPERIMENT.yaml

* Update drugprot.py

* Update pipeline.py

* with pre-commit

* with pre-commit

* Create PN_010423_concatenated_classification.yaml

* corrected

* added ultralytics detection adapter

ultralytics detection adapter with dataset and experiment configs

* Update ultralytics_adapter.py

* Update ultralytics_adapter.py

* Update NP_210523_yolov8_insects.yaml

* Rename PN_010423_concatenated_classification.yaml to NP_010423_concatenated_classification.yaml

* added ultralytics package

* fixed a bug

* fixed a bug

* style changes

* fixed a bug

* fixed a bug

* style changes

* bugfixes

* updated folder structure for yolo

* fixed imports

* bugfixes and refactoring

* bugfixes

* changed data

* changed data

* updated test

* bugfix

---------

Co-authored-by: Kazybek Askarbek <[email protected]>

config/datasets/detection/detection_ar_tax_or.yaml

@@ -7,7 +7,7 @@ description: Dataset for image detection
 markup_info: Dataset for image detection
 date_time: 19.04.2023
 
-_target_: innofw.core.integrations.ultralytics.datamodule.YOLOV5DataModuleAdapter
+_target_: innofw.core.integrations.ultralytics.datamodule.UltralyticsDataModuleAdapter
 
 
 train:

config/datasets/detection/detection_insects.yaml

@@ -0,0 +1,34 @@
+task:
+  - image-detection
+
+name: detection_insects
+description: Набор данных содержит фотографии членистоногих, включая насекомых, пауков, ракообразных, многоножек, многоножек и т.д.
+
+markup_info: Набор данных содержит разметку bounding box легких и патологий.
+date_time: 21.05.2023
+
+_target_: innofw.core.datamodules.lightning_datamodules.detection.UltralyticsDataModule
+
+train:
+  source: ./data/dataK/train/
+
+test:
+  source: ./data/dataK/test/
+
+infer:
+  source: ./data/dataK/test/
+
+num_workers: 8
+
+val_size: 0.2
+channels_num: 3
+image_size: 640
+num_classes: 7
+names:
+  - 'Hymenoptera'
+  - 'Hemiptera'
+  - 'Lepidoptera'
+  - 'Coleoptera'
+  - 'Diptera'
+  - 'Araneae'
+  - 'Odonata'

config/experiments/NP_010423_concatenated_classification.yaml

@@ -0,0 +1,19 @@
+# @package _global_
+defaults:
+  - /datasets@_dataset_dict.classification_mnist1: classification/classification_mnist
+  - /datasets@_dataset_dict.classification_mnist2: classification/classification_mnist
+  - override /models: resnet.yaml
+  - override /augmentations_train: none #classification.yaml
+  - override /augmentations_val: none
+  - override /augmentations_test: none
+  - override /losses: log_loss.yaml
+  - override /optimizers: adam
+
+project: "mnist_classification"
+task: "image-classification"
+random_seed: 42
+original_work_dir: ${hydra:runtime.cwd}
+weights_freq: 1
+batch_size: 8
+stop_param: 1
+epochs: 1

config/experiments/NP_210523_yolov8_insects.yaml

@@ -0,0 +1,21 @@
+# @package _global_
+defaults:
+  - override /models: ultralytics
+  - override /datasets: detection/detection_ar_tax_or
+  - override /loggers: wandb
+  - override /optimizers: sgd
+
+project: "insects"
+task: "image-detection"
+random_seed: 43
+epochs: 100
+batch_size: 4
+weights_freq: 1
+n_classes: 7
+imgsz: 640
+optimizers:
+  name: SGD
+schedulers:
+  lr0: 0.0001
+  lrf: 0.01 
+#ckpt_path: 'C:\\Users\\nikit\\Downloads\\innofw-main\\innofw-main\\logs\\something\\20230522-125003\\weights\\best.pt'

config/models/detection/yolov5.yaml

@@ -1,8 +1,8 @@
 name: yolov5
 description: yolov5 by ultralytics
 
-_target_: innofw.core.models.torch.architectures.detection.YOLOv5
-arch: yolov5m  # n s m l x
+_target_: ultralytics.YOLO
+model: yolov5mu  # n s m l x
 
 
 #--weights yolov5x.pt

config/models/ultralytics.yaml

@@ -0,0 +1,7 @@
+name: ultralytics
+description: ultralytics
+
+_target_: ultralytics.YOLO
+
+model: yolov8s.pt
+

config/predict.yaml

@@ -0,0 +1,35 @@
+defaults:
+  - _self_
+  - datasets: 
+  - models: 
+  - trainer: auto
+  - metrics: auto
+  - losses: auto
+  - optimizers: auto
+  - schedulers: auto
+  - augmentations_train: none
+  - augmentations_test: none
+  - augmentations_val: none
+  - loggings: auto
+  - callbacks: auto
+  - initializations:
+  - wandb: default
+  - loggers: none
+  - experiments:  # order matters
+  - clear_ml: disabled
+  - override hydra/job_logging: disabled
+#  - override hydra/job_logging: custom
+
+project: 
+task: 
+random_seed:
+epochs:
+batch_size:
+weights_path:
+weights_freq:
+ckpt_path:
+stop_param:
+experiment_name:
+
+accelerator: gpu
+devices: 1

config/test.yaml

@@ -13,12 +13,12 @@ defaults:
   - loggings: auto
   - callbacks: auto
   - initializations:
-  - experiments: ???
+  - experiments: 
   - clear_ml: disabled
   - override hydra/job_logging: custom
 
-project: ???
-task: ???
+project: 
+task: 
 seed:
 epochs:
 batch_size:
@@ -27,7 +27,7 @@ weights_freq:
 ckpt_path:
 experiment_name:
 
-accelerator: cpu
+accelerator: gpu
 #devices: 1
 #gpus: 1
 

innofw/constants.py

@@ -1,13 +1,10 @@
 # standard libraries
 from enum import Enum
 from pathlib import Path
-from typing import Optional
-from typing import TypeVar
-
-from pydantic import BaseModel
-from pydantic import SecretStr
+from typing import TypeVar, Optional
 
 # third-party libraries
+from pydantic import BaseModel, SecretStr
 
 
 class Stages(Enum):
@@ -21,9 +18,9 @@ class Frameworks(Enum):
     sklearn = "sklearn"
     xgboost = "xgboost"
     catboost = "catboost"
+    ultralytics = "ultralytics"
     none = "none"
 
-
 PathLike = TypeVar(
     "PathLike", str, Path
 )  # ref: https://stackoverflow.com/questions/58541722/what-is-the-correct-way-in-python-to-annotate-a-path-with-type-hints
@@ -91,17 +88,16 @@ class CLI_FLAGS(Enum):
 
 DEFAULT_STORAGE_URL: str = "https://api.blackhole.ai.innopolis.university:443"
 
-
 # === semantic segmentation ===
 # constants
 class SegDataKeys(Enum):
-    image = "image"
-    label = "label"
-    filename = "name"
-    coords = "coords"
-    metadata = "metadata"
+    image = 'image'
+    label = 'label'
+    filename = 'name'
+    coords = 'coords'
+    metadata = 'metadata'
 
 
 class SegOutKeys(Enum):  # todo: use it somehow
     predictions = "predictions"
-    metrics = "metrics"
+    metrics = "metrics"

innofw/core/augmentations/__init__.py

@@ -48,8 +48,7 @@ def register_function_fn(cls):
             raise ValueError("Name %s already registered!" % name)
         if not issubclass(cls, BaseAugmentationAdapter):
             raise ValueError(
-                "Class %s is not a subclass of %s"
-                % (cls, BaseAugmentationAdapter)
+                "Class %s is not a subclass of %s" % (cls, BaseAugmentationAdapter)
             )
         __AUG_ADAP_DICT__[name] = cls
         return cls
@@ -60,9 +59,7 @@ def register_function_fn(cls):
 for file in os.listdir(os.path.dirname(__file__)):
     if file.endswith(".py") and not file.startswith("_"):
         module_name = file[: file.find(".py")]
-        module = importlib.import_module(
-            "innofw.core.augmentations." + module_name
-        )
+        module = importlib.import_module("innofw.core.augmentations." + module_name)
 
 
 class Augmentation(nn.Module):
@@ -81,9 +78,7 @@ class Augmentation(nn.Module):
 
     def __init__(self, augmentations):
         super().__init__()
-        self.augs = (
-            None if augmentations is None else get_augs_adapter(augmentations)
-        )
+        self.augs = None if augmentations is None else get_augs_adapter(augmentations)
 
     def forward(self, x, y=None):
         if self.augs is None:

innofw/core/augmentations/albumentations_adapter.py

@@ -33,10 +33,7 @@ def __init__(self, transforms, *args, **kwargs):
     def forward(self, x, y=None, z=None):
         if y is not None:
             result = self.transforms(image=np.array(x), mask=y)
-            if (
-                len(result["image"].shape) == 3
-                and result["image"].shape[2] == 3
-            ):
+            if len(result["image"].shape) == 3 and result["image"].shape[2] == 3:
                 img = np.moveaxis(result["image"], -1, 0)
             else:
                 img = result["image"]

innofw/core/augmentations/torchvision_adapter.py

@@ -30,9 +30,9 @@ def __init__(self, transforms, *args, **kwargs):
 
     @staticmethod
     def is_suitable_input(transforms) -> bool:
-        return isinstance(
-            transforms, torchvision.transforms.Compose
-        ) or isinstance(transforms, nn.Module)
+        return isinstance(transforms, torchvision.transforms.Compose) or isinstance(
+            transforms, nn.Module
+        )
 
     def forward(self, x):
         return self.transforms(np.array(x))

innofw/core/datamodules/lightning_datamodules/concatenated_datamodule.py

@@ -0,0 +1,122 @@
+
+from torch.utils.data import DataLoader
+from innofw.constants import Frameworks
+from innofw.core.datamodules.lightning_datamodules.base import BaseLightningDataModule
+from pydantic import validate_arguments
+from torch.utils.data import ConcatDataset
+from innofw.constants import Frameworks, Stages
+from innofw.core.augmentations import Augmentation
+import logging
+import os.path
+import pathlib
+import pandas as pd
+from innofw.core.datamodules.lightning_datamodules.drugprot import DataCollatorWithPaddingAndTruncation
+from torch.utils.data import WeightedRandomSampler
+import numpy as np
+
+def collate_fn(batch):
+    return tuple(zip(*batch))
+
+class ConcatenatedLightningDatamodule(BaseLightningDataModule):
+
+    framework = [Frameworks.torch]
+
+    @validate_arguments
+    def __init__(
+        self,
+        class_name,
+        datamodules,
+        batch_size: int = 4,
+        val_size: float = 0.2,
+        num_workers: int = 1,
+        augmentations=None,
+        infer=None,
+        stage=None,
+        channels_num = None,
+        random_seed = None,
+        w_sampler = False,
+        *args,
+        **kwargs
+    ):
+        self.datamodules = datamodules 
+        self.batch_size = batch_size
+        self.shuffle = True
+        self.num_workers = num_workers
+        self.stage = stage
+        self.infer = infer
+        self.val_size = val_size
+        self.augmentations=augmentations
+        self.class_name = class_name
+        self.channels_num = channels_num
+        self.val_size = val_size
+        self.random_seed = random_seed
+        self.w_sampler = w_sampler
+        self.mul = 1
+        self.batch_sampler = None
+        self.collate_fn = None
+        self.pin_memory = False
+        self.worker_init_fn = None
+        self.prefetch_factor = 1
+        self.persistent_workers = False
+        self.timeout = 0
+        self.shuffle = False
+        self.drop_last = False
+
+    def setup_train_test_val(self, **kwargs):
+        [dm.setup_train_test_val() for dm in self.datamodules]
+        self.train_ds = ConcatDataset([dm.train_dataset for dm in self.datamodules])
+        #self.test_ds = ConcatDataset([dm.test_dataset for dm in self.datamodules])
+        self.val_ds = ConcatDataset([dm.val_dataset for dm in self.datamodules])
+
+    def stage_dataloader(self, dataset, stage):
+        if stage in ["val", "test", "predict"]:
+            self.shuffle = False
+            self.drop_last = False
+        else:
+            self.shuffle = self.shuffle
+            self.drop_last = True
+
+        if self.class_name == "CocoLightningDataModule":
+           self.collate_fn = collate_fn
+        elif self.class_name == "DrugprotDataModule":
+            self.collate_fn = DataCollatorWithPaddingAndTruncation(
+            max_length=512,
+            sequence_keys=["input_ids", "labels"]
+        )   
+        elif self.class_name == "ImageLightningDataModule":
+            self.prefetch_factor = 2
+        elif self.class_name == "SegmentationDM":
+            self.prefetch_factor = 2
+            self.pin_memory = True
+            self.prefetch_factor = 2
+
+        return DataLoader(
+            dataset,
+            batch_size=self.batch_size,
+            shuffle=self.shuffle,
+            drop_last=self.drop_last,
+            num_workers=self.num_workers,
+            collate_fn=self.collate_fn,
+            pin_memory=self.pin_memory,
+            timeout=self.timeout,
+            worker_init_fn=self.worker_init_fn,
+            prefetch_factor=self.prefetch_factor,
+            persistent_workers=self.persistent_workers,
+            #sampler=sampler,
+            #batch_sampler=self.batch_sampler,
+        )       
+
+    def train_dataloader(self):
+        return self.stage_dataloader(self.train_ds, "train")
+
+    def val_dataloader(self):
+        return self.stage_dataloader(self.val_ds, "val")
+
+    def test_dataloader(self):
+        return self.stage_dataloader(self.test_ds, "test")
+
+    def predict_dataloader(self):
+        return self.stage_dataloader(self.predict_source, "predict")
+
+    def save_preds(self, preds, stage: Stages, dst_path: pathlib.Path):
+        pass