From 14c7444ee6b2a5780141dd3ac51bb719ad07432e Mon Sep 17 00:00:00 2001
From: Stefaan Lippens <stefaan.lippens@vito.be>
Date: Thu, 25 Jul 2024 00:36:40 +0200
Subject: [PATCH] Issue #5 apply `upload_assets` on real benchmark

---
 .github/workflows/benchmarks.yaml                  |  3 ++-
 qa/benchmarks/tests/test_benchmarks.py             | 14 +++++++++++---
 .../pytest_upload_assets.py                        |  9 +++++----
 3 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml
index 2a4cf62..8890f14 100644
--- a/.github/workflows/benchmarks.yaml
+++ b/.github/workflows/benchmarks.yaml
@@ -24,7 +24,8 @@ jobs:
           mkdir report
           mkdir tmp_path_root
           pytest \
-            --dummy \
+            --random-subset=1 \
+            -k max_ndvi_fail_intentionally \
             --html report/report.html --self-contained-html \
             --track-metrics-report=report/metrics.json \
             --basetemp=tmp_path_root
diff --git a/qa/benchmarks/tests/test_benchmarks.py b/qa/benchmarks/tests/test_benchmarks.py
index 0b952d0..1c93e80 100644
--- a/qa/benchmarks/tests/test_benchmarks.py
+++ b/qa/benchmarks/tests/test_benchmarks.py
@@ -19,7 +19,11 @@
     ],
 )
 def test_run_benchmark(
-    scenario: BenchmarkScenario, connection_factory, tmp_path: Path, track_metric
+    scenario: BenchmarkScenario,
+    connection_factory,
+    tmp_path: Path,
+    track_metric,
+    upload_assets,
 ):
     connection: openeo.Connection = connection_factory(url=scenario.backend)
 
@@ -39,8 +43,12 @@ def test_run_benchmark(
 
     # Download actual results
     actual_dir = tmp_path / "actual"
-    job.get_results().download_files(target=actual_dir, include_stac_metadata=True)
-    # TODO: upload actual results to somewhere?
+    paths = job.get_results().download_files(
+        target=actual_dir, include_stac_metadata=True
+    )
+
+    # Upload assets on failure
+    upload_assets(*paths)
 
     # Compare actual results with reference data
     reference_dir = download_reference_data(
diff --git a/qa/tools/apex_algorithm_qa_tools/pytest_upload_assets.py b/qa/tools/apex_algorithm_qa_tools/pytest_upload_assets.py
index a613395..ee82994 100644
--- a/qa/tools/apex_algorithm_qa_tools/pytest_upload_assets.py
+++ b/qa/tools/apex_algorithm_qa_tools/pytest_upload_assets.py
@@ -110,9 +110,10 @@ def upload_assets(pytestconfig, tmp_path) -> Callable[[Path], None]:
     """
     uploader = pytestconfig.pluginmanager.get_plugin(_PLUGIN_NAME)
 
-    def collect(path: Path):
-        assert path.is_relative_to(tmp_path)
-        name = str(path.relative_to(tmp_path))
-        uploader.collector.collect(path=path, name=name)
+    def collect(*paths: Path):
+        for path in paths:
+            assert path.is_relative_to(tmp_path)
+            name = str(path.relative_to(tmp_path))
+            uploader.collector.collect(path=path, name=name)
 
     return collect