diff --git a/pyproject.toml b/pyproject.toml index 19cb21c66a..4a457ee43c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dev = [ "pytest-mock", "pytest-csv", "pytest-cov", + "onnxruntime==1.20.0", ] docs = [ diff --git a/src/otx/algo/detection/atss.py b/src/otx/algo/detection/atss.py index 1a35a4ffee..92cffa60c4 100644 --- a/src/otx/algo/detection/atss.py +++ b/src/otx/algo/detection/atss.py @@ -183,7 +183,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, }, diff --git a/src/otx/algo/detection/rtmdet.py b/src/otx/algo/detection/rtmdet.py index 19c39db9cc..839c62b850 100644 --- a/src/otx/algo/detection/rtmdet.py +++ b/src/otx/algo/detection/rtmdet.py @@ -138,7 +138,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, }, diff --git a/src/otx/algo/detection/ssd.py b/src/otx/algo/detection/ssd.py index 2c3e8b01b6..28a6b8b27e 100644 --- a/src/otx/algo/detection/ssd.py +++ b/src/otx/algo/detection/ssd.py @@ -346,7 +346,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, }, diff --git a/src/otx/algo/detection/yolov9.py b/src/otx/algo/detection/yolov9.py index 092782bb3e..4f69f8ffa0 100644 --- a/src/otx/algo/detection/yolov9.py +++ b/src/otx/algo/detection/yolov9.py @@ -125,7 +125,7 @@ def _exporter(self) -> OTXModelExporter: "export_params": True, "opset_version": 11, "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, }, diff --git a/src/otx/algo/detection/yolox.py b/src/otx/algo/detection/yolox.py index 56b4f368b7..9dde070ddd 100644 --- a/src/otx/algo/detection/yolox.py +++ b/src/otx/algo/detection/yolox.py @@ -158,7 +158,7 @@ def _exporter(self) -> OTXModelExporter: "export_params": True, "opset_version": 11, "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, }, diff --git a/src/otx/algo/instance_segmentation/maskdino.py b/src/otx/algo/instance_segmentation/maskdino.py index d468a09ca9..c77f4fdc27 100644 --- a/src/otx/algo/instance_segmentation/maskdino.py +++ b/src/otx/algo/instance_segmentation/maskdino.py @@ -190,7 +190,7 @@ def _exporter(self) -> OTXModelExporter: onnx_export_configuration={ "input_names": ["image"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, "masks": {0: "batch", 1: "num_dets", 2: "height", 3: "width"}, diff --git a/src/otx/algo/instance_segmentation/maskrcnn.py b/src/otx/algo/instance_segmentation/maskrcnn.py index 138df2366c..ee449eeb05 100644 --- a/src/otx/algo/instance_segmentation/maskrcnn.py +++ b/src/otx/algo/instance_segmentation/maskrcnn.py @@ -319,7 +319,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels", "masks"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, "masks": {0: "batch", 1: "num_dets", 2: "height", 3: "width"}, diff --git a/src/otx/algo/instance_segmentation/maskrcnn_tv.py b/src/otx/algo/instance_segmentation/maskrcnn_tv.py index 0647488b43..48697b6b4a 100644 --- a/src/otx/algo/instance_segmentation/maskrcnn_tv.py +++ b/src/otx/algo/instance_segmentation/maskrcnn_tv.py @@ -214,7 +214,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels", "masks"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, "masks": {0: "batch", 1: "num_dets", 2: "height", 3: "width"}, diff --git a/src/otx/algo/instance_segmentation/rtmdet_inst.py b/src/otx/algo/instance_segmentation/rtmdet_inst.py index 3625d46f87..baa03d23d4 100644 --- a/src/otx/algo/instance_segmentation/rtmdet_inst.py +++ b/src/otx/algo/instance_segmentation/rtmdet_inst.py @@ -122,7 +122,7 @@ def _exporter(self) -> OTXModelExporter: "input_names": ["image"], "output_names": ["boxes", "labels", "masks"], "dynamic_axes": { - "image": {0: "batch", 2: "height", 3: "width"}, + "image": {0: "batch"}, "boxes": {0: "batch", 1: "num_dets"}, "labels": {0: "batch", 1: "num_dets"}, "masks": {0: "batch", 1: "num_dets", 2: "height", 3: "width"}, diff --git a/tests/integration/cli/test_export_inference.py b/tests/integration/cli/test_export_inference.py index 1d455616c4..5b0035f016 100644 --- a/tests/integration/cli/test_export_inference.py +++ b/tests/integration/cli/test_export_inference.py @@ -6,6 +6,7 @@ from copy import copy from pathlib import Path +import onnxruntime as ort import pandas as pd import pytest @@ -53,12 +54,13 @@ def fxt_local_seed() -> int: } +# TODO(someone): this test is too complex and should be split into multiple tests. @pytest.mark.parametrize( "recipe", pytest.RECIPE_LIST, ids=lambda x: "/".join(Path(x).parts[-2:]), ) -def test_otx_export_infer( +def test_otx_export_infer( # noqa: C901 recipe: str, tmp_path: Path, fxt_local_seed: int, @@ -142,7 +144,7 @@ def test_otx_export_infer( assert len(ckpt_files) > 0 # 2) otx test - def run_cli_test( + def __run_cli_test( test_recipe: str, checkpoint_path: str, work_dir: Path, @@ -188,7 +190,7 @@ def run_cli_test( return tmp_path_test checkpoint_path: str = str(ckpt_files[-1]) - tmp_path_test = run_cli_test(recipe, checkpoint_path, Path("outputs") / "torch", with_benchmark=True) + tmp_path_test = __run_cli_test(recipe, checkpoint_path, Path("outputs") / "torch", with_benchmark=True) if task == "zero_shot_visual_prompting": # Check when using reference infos obtained by otx train @@ -198,7 +200,7 @@ def run_cli_test( str(Path(checkpoint_path).parents[-idx_task] / f"otx_train_{model_name}/outputs/.latest/train"), ] - tmp_path_test = run_cli_test( + tmp_path_test = __run_cli_test( recipe, checkpoint_path, Path("outputs") / "torch", @@ -209,10 +211,10 @@ def run_cli_test( assert (tmp_path_test / "outputs" / "torch" / ".latest" / "benchmark" / "benchmark_report.csv").exists() # 3) otx export - format_to_ext = {"OPENVINO": "xml"} # [TODO](@Vlad): extend to "ONNX": "onnx" + format_to_ext = {"OPENVINO": "xml", "ONNX": "onnx"} tmp_path_test = tmp_path / f"otx_test_{model_name}" - for fmt in format_to_ext: + for fmt, ext in format_to_ext.items(): command_cfg = [ "otx", "export", @@ -238,128 +240,142 @@ def run_cli_test( ) assert latest_dir.exists() if task in ("visual_prompting", "zero_shot_visual_prompting"): - assert (latest_dir / f"exported_model_image_encoder.{format_to_ext[fmt]}").exists() - assert (latest_dir / f"exported_model_decoder.{format_to_ext[fmt]}").exists() + assert (latest_dir / f"exported_model_image_encoder.{ext}").exists() + assert (latest_dir / f"exported_model_decoder.{ext}").exists() else: - assert (latest_dir / f"exported_model.{format_to_ext[fmt]}").exists() - - # 4) infer of the exported models - task = recipe.split("/")[-2] - tmp_path_test = tmp_path / f"otx_test_{model_name}" - if "_cls" in recipe: - export_test_recipe = f"src/otx/recipe/classification/{task}/openvino_model.yaml" - else: - export_test_recipe = f"src/otx/recipe/{task}/openvino_model.yaml" - - if task in ("visual_prompting", "zero_shot_visual_prompting"): - exported_model_path = str(latest_dir / "exported_model_decoder.xml") - else: - exported_model_path = str(latest_dir / "exported_model.xml") - - tmp_path_test = run_cli_test(export_test_recipe, exported_model_path, Path("outputs") / "openvino", "cpu") - assert (tmp_path_test / "outputs").exists() - - if task == "zero_shot_visual_prompting": - # Check when using reference infos obtained by otx train - idx_task = exported_model_path.split("/").index(f"otx_test_{model_name}") - infer_reference_info_root = [ - "--model.init_args.infer_reference_info_root", - str(Path(exported_model_path).parents[-idx_task] / f"otx_train_{model_name}/outputs/.latest/train"), - ] - tmp_path_test = run_cli_test( - export_test_recipe, - exported_model_path, - Path("outputs") / "openvino", - "cpu", - cli_override_command=infer_reference_info_root, - ) - - # 5) test optimize - command_cfg = [ - "otx", - "optimize", - "--config", - export_test_recipe, - "--data_root", - fxt_target_dataset_per_task[task], - "--work_dir", - str(tmp_path_test / "outputs"), - "--engine.device", - "cpu", - *fxt_cli_override_command_per_task[task], - "--checkpoint", - exported_model_path, - ] - - run_main(command_cfg=command_cfg, open_subprocess=fxt_open_subprocess) - - outputs_dir = tmp_path_test / "outputs" - latest_dir = max( - (p for p in outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), - key=lambda p: p.stat().st_mtime, - ) - assert latest_dir.exists() - if task in ("visual_prompting", "zero_shot_visual_prompting"): - optimized_model_path = str(latest_dir / "optimized_model_decoder.xml") - else: - optimized_model_path = str(latest_dir / "optimized_model.xml") - - # 6) test optimized model - tmp_path_test = run_cli_test(export_test_recipe, optimized_model_path, Path("outputs") / "nncf_ptq", "cpu") - if task == "zero_shot_visual_prompting": - # Check when using reference infos obtained by otx train - idx_task = optimized_model_path.split("/").index(f"otx_test_{model_name}") - infer_reference_info_root = [ - "--model.init_args.infer_reference_info_root", - str(Path(optimized_model_path).parents[-idx_task] / f"otx_train_{model_name}/outputs/.latest/train"), - ] - tmp_path_test = run_cli_test( - export_test_recipe, - optimized_model_path, - Path("outputs") / "nncf_ptq", - "cpu", - cli_override_command=infer_reference_info_root, - ) - - torch_outputs_dir = tmp_path_test / "outputs" / "torch" - torch_latest_dir = max( - (p for p in torch_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), - key=lambda p: p.stat().st_mtime, - ) - openvino_outputs_dir = tmp_path_test / "outputs" / "openvino" - openvino_latest_dir = max( - (p for p in openvino_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), - key=lambda p: p.stat().st_mtime, - ) - nncf_ptq_outputs_dir = tmp_path_test / "outputs" / "nncf_ptq" - nncf_ptq_latest_dir = max( - (p for p in nncf_ptq_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), - key=lambda p: p.stat().st_mtime, - ) - assert nncf_ptq_latest_dir.exists() - - df_torch = pd.read_csv(next(torch_latest_dir.glob("**/metrics.csv"))) - df_openvino = pd.read_csv(next(openvino_latest_dir.glob("**/metrics.csv"))) - df_nncf_ptq = pd.read_csv(next(nncf_ptq_latest_dir.glob("**/metrics.csv"))) - - metric_name = TASK_NAME_TO_MAIN_METRIC_NAME[task] - - assert metric_name in df_torch.columns - assert metric_name in df_openvino.columns - assert metric_name in df_nncf_ptq.columns - - torch_acc = df_torch[metric_name].item() - ov_acc = df_openvino[metric_name].item() - ptq_acc = df_nncf_ptq[metric_name].item() - - msg = f"Recipe: {recipe}, (torch_accuracy, ov_accuracy, ptq_acc): {torch_acc}, {ov_acc}, {ptq_acc}" - log.info(msg) - - # Not compare w/ instance segmentation and visual prompting tasks because training isn't able to be deterministic, which can lead to unstable test result. - if "maskrcnn_efficientnetb2b" in recipe or task in ("visual_prompting", "zero_shot_visual_prompting"): - return - - # This test seems fragile, so that disable it. - # Model accuracy should be checked at the regression tests - # https://github.com/openvinotoolkit/training_extensions/actions/runs/8340264268/job/22824202673?pr=3155 - # _check_relative_metric_diff(torch_acc, ov_acc, threshold) noqa: ERA001 + assert (latest_dir / f"exported_model.{ext}").exists() + + if fmt == "ONNX": + onnx_model_path = latest_dir / f"exported_model.{ext}" + try: + ort.InferenceSession(str(onnx_model_path)) + except Exception: + log.exception("ONNX model %s is invalid", model_name) + else: + # 4) infer of the exported models + task = recipe.split("/")[-2] + tmp_path_test = tmp_path / f"otx_test_{model_name}" + if "_cls" in recipe: + export_test_recipe = f"src/otx/recipe/classification/{task}/openvino_model.yaml" + else: + export_test_recipe = f"src/otx/recipe/{task}/openvino_model.yaml" + + if task in ("visual_prompting", "zero_shot_visual_prompting"): + exported_model_path = str(latest_dir / "exported_model_decoder.xml") + else: + exported_model_path = str(latest_dir / "exported_model.xml") + + tmp_path_test = __run_cli_test(export_test_recipe, exported_model_path, Path("outputs") / "openvino", "cpu") + assert (tmp_path_test / "outputs").exists() + + if task == "zero_shot_visual_prompting": + # Check when using reference infos obtained by otx train + idx_task = exported_model_path.split("/").index(f"otx_test_{model_name}") + infer_reference_info_root = [ + "--model.init_args.infer_reference_info_root", + str(Path(exported_model_path).parents[-idx_task] / f"otx_train_{model_name}/outputs/.latest/train"), + ] + tmp_path_test = __run_cli_test( + export_test_recipe, + exported_model_path, + Path("outputs") / "openvino", + "cpu", + cli_override_command=infer_reference_info_root, + ) + + # 5) test optimize + command_cfg = [ + "otx", + "optimize", + "--config", + export_test_recipe, + "--data_root", + fxt_target_dataset_per_task[task], + "--work_dir", + str(tmp_path_test / "outputs"), + "--engine.device", + "cpu", + *fxt_cli_override_command_per_task[task], + "--checkpoint", + exported_model_path, + ] + + run_main(command_cfg=command_cfg, open_subprocess=fxt_open_subprocess) + + outputs_dir = tmp_path_test / "outputs" + latest_dir = max( + (p for p in outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), + key=lambda p: p.stat().st_mtime, + ) + assert latest_dir.exists() + if task in ("visual_prompting", "zero_shot_visual_prompting"): + optimized_model_path = str(latest_dir / "optimized_model_decoder.xml") + else: + optimized_model_path = str(latest_dir / "optimized_model.xml") + + # 6) test optimized model + tmp_path_test = __run_cli_test( + export_test_recipe, + optimized_model_path, + Path("outputs") / "nncf_ptq", + "cpu", + ) + if task == "zero_shot_visual_prompting": + # Check when using reference infos obtained by otx train + idx_task = optimized_model_path.split("/").index(f"otx_test_{model_name}") + infer_reference_info_root = [ + "--model.init_args.infer_reference_info_root", + str( + Path(optimized_model_path).parents[-idx_task] / f"otx_train_{model_name}/outputs/.latest/train", + ), + ] + tmp_path_test = __run_cli_test( + export_test_recipe, + optimized_model_path, + Path("outputs") / "nncf_ptq", + "cpu", + cli_override_command=infer_reference_info_root, + ) + + torch_outputs_dir = tmp_path_test / "outputs" / "torch" + torch_latest_dir = max( + (p for p in torch_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), + key=lambda p: p.stat().st_mtime, + ) + openvino_outputs_dir = tmp_path_test / "outputs" / "openvino" + openvino_latest_dir = max( + (p for p in openvino_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), + key=lambda p: p.stat().st_mtime, + ) + nncf_ptq_outputs_dir = tmp_path_test / "outputs" / "nncf_ptq" + nncf_ptq_latest_dir = max( + (p for p in nncf_ptq_outputs_dir.iterdir() if p.is_dir() and p.name != ".latest"), + key=lambda p: p.stat().st_mtime, + ) + assert nncf_ptq_latest_dir.exists() + + df_torch = pd.read_csv(next(torch_latest_dir.glob("**/metrics.csv"))) + df_openvino = pd.read_csv(next(openvino_latest_dir.glob("**/metrics.csv"))) + df_nncf_ptq = pd.read_csv(next(nncf_ptq_latest_dir.glob("**/metrics.csv"))) + + metric_name = TASK_NAME_TO_MAIN_METRIC_NAME[task] + + assert metric_name in df_torch.columns + assert metric_name in df_openvino.columns + assert metric_name in df_nncf_ptq.columns + + torch_acc = df_torch[metric_name].item() + ov_acc = df_openvino[metric_name].item() + ptq_acc = df_nncf_ptq[metric_name].item() + + msg = f"Recipe: {recipe}, (torch_accuracy, ov_accuracy, ptq_acc): {torch_acc}, {ov_acc}, {ptq_acc}" + log.info(msg) + + # Not compare w/ instance segmentation and visual prompting tasks because training isn't able to be deterministic, which can lead to unstable test result. + if "maskrcnn_efficientnetb2b" in recipe or task in ("visual_prompting", "zero_shot_visual_prompting"): + return + + # This test seems flaky, so let's disable it. + # Model accuracy should be checked at the regression tests + # https://github.com/openvinotoolkit/training_extensions/actions/runs/8340264268/job/22824202673?pr=3155 + # _check_relative_metric_diff(torch_acc, ov_acc, threshold) noqa: ERA001