Update benchmark datasets (#4068)

* Update benchmark datasets * Fix pre-commit issue
openvinotoolkit · Oct 28, 2024 · 662f9ca · 662f9ca
1 parent 0d87ca6
commit 662f9ca
Show file tree

Hide file tree

Showing 3 changed files with 72 additions and 42 deletions.
diff --git a/tests/perf/test_classification.py b/tests/perf/test_classification.py
@@ -57,8 +57,8 @@ class TestPerfSingleLabelClassification(PerfTestBase):
             extra_overrides={},
         ),
         Benchmark.Dataset(
-            name="multiclass_food101_large",
-            path=Path("multiclass_classification/multiclass_food101_large"),
+            name="multiclass_food20_large",
+            path=Path("multiclass_classification/multiclass_food20_large"),
             group="large",
             num_repeat=5,
             extra_overrides={},
@@ -139,8 +139,8 @@ class TestPerfMultiLabelClassification(PerfTestBase):
             extra_overrides={},
         ),
         Benchmark.Dataset(
-            name="multilabel_food101_large",
-            path=Path("multilabel_classification/multilabel_food101_large"),
+            name="multilabel_food20_large",
+            path=Path("multilabel_classification/multilabel_food20_large"),
             group="large",
             num_repeat=5,
             extra_overrides={},
@@ -220,7 +220,13 @@ class TestPerfHierarchicalLabelClassification(PerfTestBase):
             num_repeat=5,
             extra_overrides={},
         ),
-        # Add large dataset
+        Benchmark.Dataset(
+            name="cifar100_label_group_datum_format_large",
+            path=Path("hlabel_classification/cifar100_label_group_datum_format_large"),
+            group="large",
+            num_repeat=5,
+            extra_overrides={},
+        ),
     ]
 
     BENCHMARK_CRITERIA = [  # noqa: RUF012
@@ -273,14 +279,38 @@ class TestPerfSemiSLMultiClass(PerfTestBase):
     """Benchmark single-label classification for Semi-SL task."""
 
     MODEL_TEST_CASES = [  # noqa: RUF012
-        Benchmark.Model(task="classification/multi_class_cls", name="efficientnet_b0_semisl", category="balance"),
-        Benchmark.Model(task="classification/multi_class_cls", name="mobilenet_v3_large_semisl", category="speed"),
-        Benchmark.Model(task="classification/multi_class_cls", name="efficientnet_v2_semisl", category="accuracy"),
-        Benchmark.Model(task="classification/multi_class_cls", name="deit_tiny_semisl", category="other"),
-        Benchmark.Model(task="classification/multi_class_cls", name="dino_v2_semisl", category="other"),
-        Benchmark.Model(task="classification/multi_class_cls", name="tv_efficientnet_b3_semisl", category="other"),
-        Benchmark.Model(task="classification/multi_class_cls", name="tv_efficientnet_v2_l_semisl", category="other"),
-        Benchmark.Model(task="classification/multi_class_cls", name="tv_mobilenet_v3_small_semisl", category="other"),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="efficientnet_b0_semisl",
+            category="balance",
+        ),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="mobilenet_v3_large_semisl",
+            category="speed",
+        ),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="efficientnet_v2_semisl",
+            category="accuracy",
+        ),
+        Benchmark.Model(task="classification/multi_class_cls/semisl", name="deit_tiny_semisl", category="other"),
+        Benchmark.Model(task="classification/multi_class_cls/semisl", name="dino_v2_semisl", category="other"),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="tv_efficientnet_b3_semisl",
+            category="other",
+        ),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="tv_efficientnet_v2_l_semisl",
+            category="other",
+        ),
+        Benchmark.Model(
+            task="classification/multi_class_cls/semisl",
+            name="tv_mobilenet_v3_small_semisl",
+            category="other",
+        ),
     ]
 
     DATASET_TEST_CASES = (

diff --git a/tests/perf/test_detection.py b/tests/perf/test_detection.py
@@ -47,8 +47,8 @@ class TestPerfObjectDetection(PerfTestBase):
 
     DATASET_TEST_CASES = [
         Benchmark.Dataset(
-            name=f"pothole_small_{idx}",
-            path=Path("detection/pothole_small") / f"{idx}",
+            name=f"bdd_small_{idx}",
+            path=Path("detection/bdd_small") / f"{idx}",
             group="small",
             num_repeat=5,
             extra_overrides={
@@ -60,8 +60,8 @@ class TestPerfObjectDetection(PerfTestBase):
         for idx in (1, 2, 3)
     ] + [
         Benchmark.Dataset(
-            name="pothole_medium",
-            path=Path("detection/pothole_medium"),
+            name="bdd_medium",
+            path=Path("detection/bdd_medium"),
             group="medium",
             num_repeat=5,
             extra_overrides={
@@ -71,8 +71,8 @@ class TestPerfObjectDetection(PerfTestBase):
             },
         ),
         Benchmark.Dataset(
-            name="vitens_large",
-            path=Path("detection/vitens_large"),
+            name="bdd_large",
+            path=Path("detection/bdd_large"),
             group="large",
             num_repeat=5,
             extra_overrides={

diff --git a/tests/perf/test_semantic_segmentation.py b/tests/perf/test_semantic_segmentation.py
@@ -37,15 +37,15 @@ class TestPerfSemanticSegmentation(PerfTestBase):
         for idx in (1, 2, 3)
     ] + [
         Benchmark.Dataset(
-            name="kvasir_medium",
-            path=Path("semantic_seg/kvasir_medium"),
+            name="cityscapes_185_70_medium",
+            path=Path("semantic_seg/cityscapes_185_70_medium"),
             group="medium",
             num_repeat=5,
             extra_overrides={},
         ),
         Benchmark.Dataset(
-            name="kvasir_large",
-            path=Path("semantic_seg/kvasir_large"),
+            name="voc_2012_cut_large",
+            path=Path("semantic_seg/voc_2012_cut_large"),
             group="large",
             num_repeat=5,
             extra_overrides={},
@@ -102,46 +102,46 @@ class TestPerfSemanticSegmentationSemiSL(TestPerfSemanticSegmentation):
     """Benchmark semantic segmentation."""
 
     MODEL_TEST_CASES = [  # noqa: RUF012
-        Benchmark.Model(task="semantic_segmentation", name="litehrnet_18_semisl", category="balance"),
-        Benchmark.Model(task="semantic_segmentation", name="litehrnet_s_semisl", category="speed"),
-        Benchmark.Model(task="semantic_segmentation", name="litehrnet_x_semisl", category="accuracy"),
-        Benchmark.Model(task="semantic_segmentation", name="segnext_b_semisl", category="other"),
-        Benchmark.Model(task="semantic_segmentation", name="segnext_s_semisl", category="other"),
-        Benchmark.Model(task="semantic_segmentation", name="segnext_t_semisl", category="other"),
-        Benchmark.Model(task="semantic_segmentation", name="dino_v2_semisl", category="other"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="litehrnet_18_semisl", category="balance"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="litehrnet_s_semisl", category="speed"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="litehrnet_x_semisl", category="accuracy"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="segnext_b_semisl", category="other"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="segnext_s_semisl", category="other"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="segnext_t_semisl", category="other"),
+        Benchmark.Model(task="semantic_segmentation/semisl", name="dino_v2_semisl", category="other"),
     ]
 
     DATASET_TEST_CASES = [  # noqa: RUF012
         Benchmark.Dataset(
             name="kvasir",
-            path=Path("semantic_seg/semisl/kvasir_24"),
+            path=Path("semantic_seg/semi-sl/kvasir_24"),
             group="small",
-            num_repeat=5,
-            unlabeled_data_path=Path("semantic_seg/semisl/unlabeled_images/kvasir"),
+            num_repeat=1,
+            unlabeled_data_path=Path("semantic_seg/semi-sl/unlabeled_images/kvasir"),
             extra_overrides={},
         ),
         Benchmark.Dataset(
             name="kitti",
-            path=Path("semantic_seg/semisl/kitti_18"),
+            path=Path("semantic_seg/semi-sl/kitti_36"),
             group="small",
-            num_repeat=5,
-            unlabeled_data_path=Path("semantic_seg/semisl/unlabeled_images/kitti"),
+            num_repeat=1,
+            unlabeled_data_path=Path("semantic_seg/semi-sl/unlabeled_images/kitti"),
             extra_overrides={},
         ),
         Benchmark.Dataset(
             name="cityscapes",
-            path=Path("semantic_seg/semisl/cityscapes"),
+            path=Path("semantic_seg/semi-sl/cityscapes"),
             group="medium",
-            num_repeat=5,
-            unlabeled_data_path=Path("semantic_seg/semisl/unlabeled_images/cityscapes"),
+            num_repeat=1,
+            unlabeled_data_path=Path("semantic_seg/semi-sl/unlabeled_images/cityscapes"),
             extra_overrides={},
         ),
         Benchmark.Dataset(
             name="pascal_voc",
-            path=Path("semantic_seg/semisl/pascal_voc"),
+            path=Path("semantic_seg/semi-sl/pascal_voc"),
             group="large",
-            num_repeat=5,
-            unlabeled_data_path=Path("semantic_seg/semisl/unlabeled_images/pascal_voc"),
+            num_repeat=1,
+            unlabeled_data_path=Path("semantic_seg/semi-sl/unlabeled_images/pascal_voc"),
             extra_overrides={},
         ),
     ]