Bring RTMDetInst Optimization and tile unit test to Dev (#3195)

* Add rtmdet inst optim & Add tile unit test

src/otx/algo/instance_segmentation/heads/__init__.py

@@ -4,5 +4,6 @@
 """Custom head architecture for OTX instance segmentation models."""
 
 from .custom_roi_head import CustomConvFCBBoxHead, CustomRoIHead
+from .custom_rtmdet_ins_head import CustomRTMDetInsSepBNHead
 
-__all__ = ["CustomRoIHead", "CustomConvFCBBoxHead"]
+__all__ = ["CustomRoIHead", "CustomConvFCBBoxHead", "CustomRTMDetInsSepBNHead"]

src/otx/algo/instance_segmentation/mmconfigs/rtmdet_inst_tiny.yaml

@@ -46,7 +46,7 @@ neck:
     type: SiLU
     inplace: true
 bbox_head:
-  type: RTMDetInsSepBNHead
+  type: CustomRTMDetInsSepBNHead
   num_classes: 80
   in_channels: 96
   stacked_convs: 2

src/otx/core/data/dataset/instance_segmentation.py

@@ -55,7 +55,7 @@ def _get_item_impl(self, index: int) -> InstanceSegDataEntity | None:
                     gt_masks.append(polygon_to_bitmap([annotation], *img_shape)[0])
 
         # convert xywh to xyxy format
-        bboxes = np.array(gt_bboxes, dtype=np.float32)
+        bboxes = np.array(gt_bboxes, dtype=np.float32) if gt_bboxes else np.empty((0, 4))
         bboxes[:, 2:] += bboxes[:, :2]
 
         masks = np.stack(gt_masks, axis=0) if gt_masks else np.zeros((0, *img_shape), dtype=bool)

src/otx/core/model/detection.py

@@ -443,7 +443,7 @@ def _customize_outputs(
                 tv_tensors.BoundingBoxes(
                     output.pred_instances.bboxes,
                     format="XYXY",
-                    canvas_size=output.img_shape,
+                    canvas_size=output.ori_shape,
                 ),
             )
             labels.append(output.pred_instances.labels)

src/otx/core/utils/mask_util.py

@@ -51,7 +51,9 @@ def polygon_to_rle(
         list[dict]: List of RLE masks.
     """
     polygons = [polygon.points for polygon in polygons]
-    return mask_utils.frPyObjects(polygons, height, width)
+    if len(polygons):
+        return mask_utils.frPyObjects(polygons, height, width)
+    return []
 
 
 def encode_rle(mask: torch.Tensor) -> dict:

src/otx/recipe/instance_segmentation/openvino_model.yaml

@@ -35,4 +35,4 @@ overrides:
       image_color_channel: RGB
       data_format: coco_instances
       test_subset:
-        batch_size: 2
+        batch_size: 64

src/otx/recipe/instance_segmentation/rtmdet_inst_tiny.yaml

@@ -5,10 +5,11 @@ model:
     variant: tiny
 
 optimizer:
-  class_path: torch.optim.AdamW
+  class_path: torch.optim.SGD
   init_args:
-    lr: 0.004
-    weight_decay: 0.05
+    lr: 0.001
+    momentum: 0.9
+    weight_decay: 0.0001
 
 scheduler:
   - class_path: otx.algo.schedulers.warmup_schedulers.LinearWarmupScheduler
@@ -30,7 +31,7 @@ callback_monitor: val/map_50
 
 data: ../_base_/data/mmdet_base.yaml
 overrides:
-  precision: 32 # 16/"16-true" does not work
+  precision: 16
   max_epochs: 100
   gradient_clip_val: 35.0
   data:
@@ -39,7 +40,7 @@ overrides:
       include_polygons: true
       train_subset:
         batch_size: 4
-        num_workers: 10
+        num_workers: 8
         transforms:
           - type: LoadImageFromFile
             backend_args: null
@@ -61,6 +62,7 @@ overrides:
               - 0.5
               - 2.0
             keep_ratio: true
+            _scope_: mmdet
           - type: RandomCrop
             crop_size:
               - 640
@@ -91,8 +93,8 @@ overrides:
               - 1
           - type: PackDetInputs
       val_subset:
-        batch_size: 2
-        num_workers: 10
+        batch_size: 1
+        num_workers: 4
         transforms:
           - type: LoadImageFromFile
             backend_args: null
@@ -115,8 +117,8 @@ overrides:
               - img_shape
               - scale_factor
       test_subset:
-        batch_size: 2
-        num_workers: 10
+        batch_size: 1
+        num_workers: 4
         transforms:
           - type: LoadImageFromFile
             backend_args: null

src/otx/recipe/instance_segmentation/rtmdet_inst_tiny_tile.yaml

@@ -0,0 +1,117 @@
+model:
+  class_path: otx.algo.instance_segmentation.rtmdet_inst.RTMDetInst
+  init_args:
+    num_classes: 80
+    variant: tiny
+
+optimizer:
+  class_path: torch.optim.SGD
+  init_args:
+    lr: 0.001
+    momentum: 0.9
+    weight_decay: 0.0001
+
+scheduler:
+  - class_path: otx.algo.schedulers.warmup_schedulers.LinearWarmupScheduler
+    init_args:
+      num_warmup_steps: 20
+  - class_path: lightning.pytorch.cli.ReduceLROnPlateau
+    init_args:
+      mode: max
+      factor: 0.1
+      patience: 9
+      monitor: val/map_50
+      min_lr: 4e-06
+
+engine:
+  task: INSTANCE_SEGMENTATION
+  device: auto
+
+callback_monitor: val/map_50
+
+data: ../_base_/data/mmdet_base.yaml
+overrides:
+  precision: 32
+  max_epochs: 100
+  gradient_clip_val: 35.0
+  data:
+    task: INSTANCE_SEGMENTATION
+    config:
+      tile_config:
+        enable_tiler: true
+        enable_adaptive_tiling: true
+      include_polygons: true
+      train_subset:
+        batch_size: 4
+        num_workers: 8
+        transforms:
+          - type: LoadImageFromFile
+            backend_args: null
+          - type: LoadAnnotations
+            with_bbox: true
+            with_mask: true
+          - type: Resize
+            scale:
+              - 640
+              - 640
+            keep_ratio: false
+          - type: Pad
+            size:
+              - 640
+              - 640
+            pad_val: 114
+          - type: RandomFlip
+            prob: 0.5
+          - type: PackDetInputs
+      val_subset:
+        batch_size: 1
+        num_workers: 4
+        transforms:
+          - type: LoadImageFromFile
+            backend_args: null
+          - type: Resize
+            scale:
+              - 640
+              - 640
+            keep_ratio: true
+          - type: Pad
+            size:
+              - 640
+              - 640
+            pad_val: 114
+          - type: LoadAnnotations
+            with_bbox: true
+            with_mask: true
+          - type: PackDetInputs
+            meta_keys:
+              - img_id
+              - img_path
+              - ori_shape
+              - img_shape
+              - scale_factor
+      test_subset:
+        batch_size: 1
+        num_workers: 4
+        transforms:
+          - type: LoadImageFromFile
+            backend_args: null
+          - type: Resize
+            scale:
+              - 640
+              - 640
+            keep_ratio: true
+          - type: Pad
+            size:
+              - 640
+              - 640
+            pad_val: 114
+          - type: LoadAnnotations
+            with_bbox: true
+            with_mask: true
+          - type: PackDetInputs
+            meta_keys:
+              - img_id
+              - img_path
+              - ori_shape
+              - img_shape
+              - scale_factor