Benchmarks: re-add dummy test module (#5)

But skipped by default

qa/benchmarks/tests/conftest.py

@@ -24,16 +24,40 @@ def pytest_addoption(parser):
         type=int,
         help="Only run random selected subset benchmarks.",
     )
+    parser.addoption(
+        "--dummy",
+        action="store_true",
+        help="Toggle to only run dummy benchmarks/tests (instead of skipping them)",
+    )
 
 
-def pytest_collection_modifyitems(session, config, items):
+def pytest_ignore_collect(collection_path, config):
     """
-    Pytest plugin to select a random subset of benchmarks to run.
+    Pytest hook to ignore certain directories/files during test collection.
+    """
+    # Note: there as some subtleties about the return values of this hook,
+    # which makes the logic slightly more complex than a naive approach would suggest:
+    # - `True` means to ignore the path,
+    # - `False` means to forcefully include it regardless of other plugins,
+    # - `None` means to keep it for now, but allow other plugins to still ignore.
+    dummy_mode = bool(config.getoption("--dummy"))
+    is_dummy_path = bool("dummy" in collection_path.name)
+    if dummy_mode and not is_dummy_path:
+        return True
+    elif not dummy_mode and is_dummy_path:
+        return True
+    else:
+        return None
 
-    based on https://alexwlchan.net/til/2024/run-random-subset-of-tests-in-pytest/
+
+def pytest_collection_modifyitems(session, config, items):
+    """
+    Pytest hook to filter/reorder collected test items.
     """
-    subset_size = config.getoption("--random-subset")
 
+    # Optionally, select a random subset of benchmarks to run.
+    # based on https://alexwlchan.net/til/2024/run-random-subset-of-tests-in-pytest/
+    subset_size = config.getoption("--random-subset")
     if subset_size >= 0:
         _log.warning(
             f"Selecting random subset of {subset_size} from {len(items)} benchmarks."

qa/benchmarks/tests/test_dummy.py

@@ -0,0 +1,38 @@
+"""
+Dummy tests to to help with tooling development.
+
+Note that this test module will be skipped by default.
+Use the `--dummy` runtime option to run these tests
+and skip all other non-dummy tests.
+"""
+
+import pytest
+
+
+@pytest.mark.parametrize("y", [4, 5, 6])
+def test_tracking(track_metric, y):
+    x = 3
+    track_metric("x squared", x * x)
+    track_metric("y", y)
+    assert x + y == 8
+
+
+def test_simple_success():
+    x = 3
+    assert x + 5 == 8
+
+
+def test_simple_fail():
+    x = 3
+    assert x + 5 == "eight"
+
+
+def test_produce_files_success(tmp_path):
+    path = tmp_path / "hello.txt"
+    path.write_text("Hello, world.\n")
+
+
+def test_produce_files_fail(tmp_path):
+    path = tmp_path / "hello.txt"
+    path.write_text("Hello, world.\n")
+    assert 1 == 2