diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index f9e69a9..989bd13 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -25,6 +25,7 @@ jobs:
           mkdir tmp_path_root
           pytest \
             -vv \
+            --random-subset=1 \
             -k max_ndvi_fail_intentionally \
             --log-cli-level=INFO \
             --html report/report.html --self-contained-html \
diff --git a/qa/benchmarks/tests/conftest.py b/qa/benchmarks/tests/conftest.py
index fa751b6..d234bb9 100644
--- a/qa/benchmarks/tests/conftest.py
+++ b/qa/benchmarks/tests/conftest.py
@@ -51,19 +51,22 @@ def pytest_ignore_collect(collection_path, config):
         return None
 
 
+@pytest.hookimpl(trylast=True)
 def pytest_collection_modifyitems(session, config, items):
     """
     Pytest hook to filter/reorder collected test items.
     """
-
     # Optionally, select a random subset of benchmarks to run.
     # based on https://alexwlchan.net/til/2024/run-random-subset-of-tests-in-pytest/
+    # Note that with current pytest versions the collection/summary stats might be messed up,
+    # see https://github.com/pytest-dev/pytest/issues/12663
     subset_size = config.getoption("--random-subset")
     if subset_size >= 0:
         _log.warning(
             f"Selecting random subset of {subset_size} from {len(items)} benchmarks."
         )
-        items[:] = random.sample(items, k=subset_size)
+        if subset_size < len(items):
+            items[:] = random.sample(items, k=subset_size)
 
 
 def _get_client_credentials_env_var(url: str) -> str: