Ignore backends/arm unit tests from pytest.ini (pytorch#3831)

Summary:
The Arm tests needs to have TOSA/Vela tools installed and they are not installed by default by installed_requirements.sh Most backends/arm tests will not test anything out of the box when running all tests without it those tools installed.

To run backends/arm test you first need to install the Arm dependacies with:

$ examples/arm/setup.sh --i-agree-to-the-contained-eula

Then the test can be executed with:

$ pytest --config-file=/dev/null backends/arm/test/

For GitHub testing this is setup/executed in the unittest-arm job see .github/workflows/pull.yml for more info.

Pull Request resolved: pytorch#3831

Reviewed By: mergennachin

Differential Revision: D58138417

Pulled By: digantdesai

fbshipit-source-id: 8ff5fb8b902910d18bc86f69b8a0d4dc0930c555

backends/arm/test/common.py

@@ -5,19 +5,10 @@
 # LICENSE file in the root directory of this source tree.
 
 import os
-import shutil
 import tempfile
 
 from executorch.backends.arm.arm_backend import ArmCompileSpecBuilder
 
-# TODO: fixme! These globs are a temporary workaround. Reasoning:
-# Running the jobs in _unittest.yml will not work since that environment doesn't
-# have the vela tool, nor the tosa_reference_model tool. Hence, we need a way to
-# run what we can in that env temporarily. Long term, vela and tosa_reference_model
-# should be installed in the CI env.
-TOSA_REF_MODEL_INSTALLED = shutil.which("tosa_reference_model")
-VELA_INSTALLED = shutil.which("vela")
-
 
 def get_tosa_compile_spec(permute_memory_to_nhwc=False, custom_path=None):
     """

backends/arm/test/misc/test_debug_feats.py

@@ -110,18 +110,13 @@ def test_numerical_diff_prints(self):
             .partition()
             .to_executorch()
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            # We expect an assertion error here. Any other issues will cause the
-            # test to fail. Likewise the test will fail if the assertion error is
-            # not present.
-            try:
-                # Tolerate 0 difference => we want to trigger a numerical diff
-                tester.run_method_and_compare_outputs(atol=0, rtol=0, qtol=0)
-            except AssertionError:
-                pass  # Implicit pass test
-            else:
-                self.fail()
+        # We expect an assertion error here. Any other issues will cause the
+        # test to fail. Likewise the test will fail if the assertion error is
+        # not present.
+        try:
+            # Tolerate 0 difference => we want to trigger a numerical diff
+            tester.run_method_and_compare_outputs(atol=0, rtol=0, qtol=0)
+        except AssertionError:
+            pass  # Implicit pass test
         else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
+            self.fail()

backends/arm/test/models/test_mobilenet_v2_arm.py

@@ -49,7 +49,7 @@ class TestMobileNetV2(unittest.TestCase):
     )
 
     def test_mv2_tosa_MI(self):
-        tester = (
+        (
             ArmTester(
                 self.mv2,
                 inputs=self.model_inputs,
@@ -60,16 +60,11 @@ def test_mv2_tosa_MI(self):
             .check(list(self.all_operators))
             .partition()
             .to_executorch()
+            .run_method_and_compare_outputs()
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs()
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def test_mv2_tosa_BI(self):
-        tester = (
+        (
             ArmTester(
                 self.mv2,
                 inputs=self.model_inputs,
@@ -81,23 +76,12 @@ def test_mv2_tosa_BI(self):
             .check(list(self.operators_after_quantization))
             .partition()
             .to_executorch()
-        )
-        if common.TOSA_REF_MODEL_INSTALLED:
             # atol=1.0 is a defensive upper limit
             # TODO MLETROCH-72
             # TODO MLETROCH-149
-            tester.run_method_and_compare_outputs(
-                atol=1.0, qtol=1, inputs=self.model_inputs
-            )
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
+            .run_method_and_compare_outputs(atol=1.0, qtol=1, inputs=self.model_inputs)
+        )
 
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     def test_mv2_u55_BI(self):
         (
             ArmTester(

backends/arm/test/ops/test_add.py

@@ -58,7 +58,7 @@ def forward(self, x, y):
     def _test_add_tosa_MI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -71,18 +71,13 @@ def _test_add_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs()
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs()
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def _test_add_tosa_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -96,15 +91,9 @@ def _test_add_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(qtol=1)
         )
 
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(qtol=1)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
-
     def _test_add_u55_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
@@ -135,10 +124,6 @@ def test_add_tosa_BI(self, test_data: torch.Tensor):
         self._test_add_tosa_BI_pipeline(self.Add(), test_data)
 
     @parameterized.expand(Add.test_parameters)
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     def test_add_u55_BI(self, test_data: torch.Tensor):
         test_data = (test_data,)
         self._test_add_u55_BI_pipeline(self.Add(), test_data)
@@ -154,10 +139,6 @@ def test_add2_tosa_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         self._test_add_tosa_BI_pipeline(self.Add2(), test_data)
 
     @parameterized.expand(Add2.test_parameters)
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     def test_add2_u55_BI(self, operand1: torch.Tensor, operand2: torch.Tensor):
         test_data = (operand1, operand2)
         self._test_add_u55_BI_pipeline(self.Add2(), test_data)

backends/arm/test/ops/test_avg_pool.py

@@ -46,7 +46,7 @@ def forward(self, x):
     def _test_avgpool2d_tosa_MI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -60,18 +60,13 @@ def _test_avgpool2d_tosa_MI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs()
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs()
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def _test_avgpool2d_tosa_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -86,13 +81,8 @@ def _test_avgpool2d_tosa_BI_pipeline(
             .check_not(["executorch_exir_dialects_edge__ops_aten_avg_pool2d_default"])
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(qtol=1)
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(qtol=1)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def _test_avgpool2d_tosa_u55_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.tensor]
@@ -142,10 +132,6 @@ def test_avgpool2d_tosa_BI(
     # Expected to fail since ArmQuantizer cannot quantize a AvgPool2D layer
     # TODO(MLETORCH-93)
     @parameterized.expand(test_data_suite)
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     @unittest.expectedFailure
     def test_avgpool2d_tosa_u55_BI(
         self,

backends/arm/test/ops/test_batch_norm.py

@@ -527,7 +527,7 @@ def forward(self, x):
     def _test_batchnorm2d_tosa_MI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -552,18 +552,13 @@ def _test_batchnorm2d_tosa_MI_pipeline(
                 ]
             )
             .to_executorch()
+            .run_method_and_compare_outputs(test_data)
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(test_data)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def _test_batchnorm2d_no_stats_tosa_MI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -586,18 +581,13 @@ def _test_batchnorm2d_no_stats_tosa_MI_pipeline(
                 ]
             )
             .to_executorch()
+            .run_method_and_compare_outputs(test_data)
         )
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(test_data)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
 
     def _test_batchnorm2d_tosa_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module,
                 inputs=test_data,
@@ -623,15 +613,9 @@ def _test_batchnorm2d_tosa_BI_pipeline(
                 ]
             )
             .to_executorch()
+            .run_method_and_compare_outputs(test_data)
         )
 
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(test_data)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
-
     def _test_batchnorm2d_u55_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
@@ -724,10 +708,6 @@ def test_batchnorm2d_tosa_BI(
     @unittest.skip(
         reason="Expected to fail since ArmQuantizer cannot quantize a BatchNorm layer"
     )
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     @unittest.expectedFailure
     def test_batchnorm2d_u55_BI(
         self,

backends/arm/test/ops/test_clone.py

@@ -35,7 +35,7 @@ def forward(self, x: torch.Tensor):
     def _test_clone_tosa_MI_pipeline(
         self, module: torch.nn.Module, test_data: torch.Tensor
     ):
-        tester = (
+        (
             ArmTester(
                 module, inputs=test_data, compile_spec=common.get_tosa_compile_spec()
             )
@@ -45,19 +45,13 @@ def _test_clone_tosa_MI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(qtol=1)
         )
 
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(qtol=1)
-        else:
-            logger.warning(
-                "TOSA ref model tool not installed, skip numerical correctness tests"
-            )
-
     def _test_clone_tosa_BI_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
-        tester = (
+        (
             ArmTester(
                 module, inputs=test_data, compile_spec=common.get_tosa_compile_spec()
             )
@@ -68,15 +62,9 @@ def _test_clone_tosa_BI_pipeline(
             .partition()
             .check_count({"torch.ops.higher_order.executorch_call_delegate": 1})
             .to_executorch()
+            .run_method_and_compare_outputs(qtol=1)
         )
 
-        if common.TOSA_REF_MODEL_INSTALLED:
-            tester.run_method_and_compare_outputs(qtol=1)
-        else:
-            raise RuntimeError(
-                "TOSA ref model tool not installed and the test is an expected fail"
-            )
-
     def _test_clone_tosa_u55_pipeline(
         self, module: torch.nn.Module, test_data: Tuple[torch.Tensor]
     ):
@@ -108,9 +96,5 @@ def test_clone_tosa_BI(self, test_tensor: torch.Tensor):
     # TODO MLETROCH-125
     @parameterized.expand(Clone.test_parameters)
     @unittest.expectedFailure
-    @unittest.skipIf(
-        not common.VELA_INSTALLED,
-        "There is no point in running U55 tests if the Vela tool is not installed",
-    )
     def test_clone_u55_BI(self, test_tensor: torch.Tensor):
         self._test_clone_tosa_u55_pipeline(self.Clone(), (test_tensor,))