From 610315df2d73ad521c7576fb02126e9a3d105284 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 26 Jul 2024 16:30:14 +0200 Subject: [PATCH] upload_assets: run `random-subset` filtering as late as possible Other selection/filter mechanisms (e.g. standard `-k`) should come before final sampling (typically just 1 sample) --- .github/workflows/benchmarks.yaml | 1 + qa/benchmarks/tests/conftest.py | 7 +++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml index f9e69a9..989bd13 100644 --- a/.github/workflows/benchmarks.yaml +++ b/.github/workflows/benchmarks.yaml @@ -25,6 +25,7 @@ jobs: mkdir tmp_path_root pytest \ -vv \ + --random-subset=1 \ -k max_ndvi_fail_intentionally \ --log-cli-level=INFO \ --html report/report.html --self-contained-html \ diff --git a/qa/benchmarks/tests/conftest.py b/qa/benchmarks/tests/conftest.py index fa751b6..d234bb9 100644 --- a/qa/benchmarks/tests/conftest.py +++ b/qa/benchmarks/tests/conftest.py @@ -51,19 +51,22 @@ def pytest_ignore_collect(collection_path, config): return None +@pytest.hookimpl(trylast=True) def pytest_collection_modifyitems(session, config, items): """ Pytest hook to filter/reorder collected test items. """ - # Optionally, select a random subset of benchmarks to run. # based on https://alexwlchan.net/til/2024/run-random-subset-of-tests-in-pytest/ + # Note that with current pytest versions the collection/summary stats might be messed up, + # see https://github.com/pytest-dev/pytest/issues/12663 subset_size = config.getoption("--random-subset") if subset_size >= 0: _log.warning( f"Selecting random subset of {subset_size} from {len(items)} benchmarks." ) - items[:] = random.sample(items, k=subset_size) + if subset_size < len(items): + items[:] = random.sample(items, k=subset_size) def _get_client_credentials_env_var(url: str) -> str: