Finetune/synchronize configuration of track_metrics/upload_assets (…

…#5/#7)

qa/tools/apex_algorithm_qa_tools/pytest_track_metrics.py

@@ -43,18 +43,26 @@ def pytest_addoption(parser: pytest.Parser):
 
 
 def pytest_configure(config):
+    if hasattr(config, "workerinput"):
+        warnings.warn("`track_metrics` plugin is not supported on xdist worker nodes.")
+        return
+
     track_metrics_path = config.getoption("track_metrics_report")
-    if (
-        track_metrics_path
-        # Don't register on xdist worker nodes
-        and not hasattr(config, "workerinput")
-    ):
+    if track_metrics_path:
         config.pluginmanager.register(
             TrackMetricsReporter(path=track_metrics_path),
             name=_TRACK_METRICS_PLUGIN_NAME,
         )
 
 
+def pytest_report_header(config):
+    plugin: TrackMetricsReporter | None = config.pluginmanager.get_plugin(
+        _TRACK_METRICS_PLUGIN_NAME
+    )
+    if plugin:
+        return f"Plugin `track_metrics` is active, reporting to {plugin.path}"
+
+
 def pytest_unconfigure(config):
     if config.pluginmanager.hasplugin(_TRACK_METRICS_PLUGIN_NAME):
         config.pluginmanager.unregister(name=_TRACK_METRICS_PLUGIN_NAME)
@@ -117,7 +125,7 @@ def track_metric(
     Returns a callable that expects a metric name and value
     """
 
-    reporter: Union[TrackMetricsReporter, None] = pytestconfig.pluginmanager.get_plugin(
+    reporter: TrackMetricsReporter | None = pytestconfig.pluginmanager.get_plugin(
         _TRACK_METRICS_PLUGIN_NAME
     )
 
@@ -126,9 +134,7 @@ def track_metric(
         def append(name: str, value: Any):
             reporter.get_metrics(request.node.user_properties).append((name, value))
     else:
-        warnings.warn(
-            "The `track_metric` fixture is requested, but no output file is defined (e.g. with `--metrics-tracker-report=path/to/metrics.json`."
-        )
+        warnings.warn("Fixture `track_metric` is a no-op (incomplete set up).")
 
         def append(name: str, value: Any):
             pass

qa/tools/apex_algorithm_qa_tools/pytest_upload_assets.py

@@ -1,12 +1,37 @@
 """
 Pytest plugin to collect files generated during benchmark/test
 and upload them to S3 (e.g. on test failure).
+
+Usage:
+
+-  Enable the plugin in `conftest.py`:
+
+    ```python
+    pytest_plugins = [
+        "apex_algorithm_qa_tools.pytest_upload_assets",
+    ]
+
+-  Use the `upload_assets` fixture to register files for upload:
+
+    ```python
+    def test_dummy(upload_assets, tmp_path):
+        path = tmp_path / "dummy.txt"
+        path.write_text("dummy content")
+        upload_assets(path)
+    ```
+
+- Run the tests with:
+    - `--upload-assets-run-id=RUNID` (optional, defaults to random UUID)
+    - `--upload-assets-endpoint-url=URL`
+    - `--upload-assets-bucket=BUCKET`
+    - and env vars `UPLOAD_ASSETS_ACCESS_KEY_ID` and `UPLOAD_ASSETS_SECRET_ACCESS_KEY` set.
 """
 
 import logging
 import os
 import re
 import uuid
+import warnings
 from pathlib import Path
 from typing import Callable, Dict, Union
 
@@ -41,12 +66,7 @@ def pytest_configure(config: pytest.Config):
     run_id = config.getoption("upload_assets_run_id")
     endpoint_url = config.getoption("upload_assets_endpoint_url")
     bucket = config.getoption("upload_assets_bucket")
-    if (
-        endpoint_url
-        and bucket
-        # Don't register on xdist worker nodes
-        and not hasattr(config, "workerinput")
-    ):
+    if endpoint_url and bucket:
         s3_client = boto3.client(
             service_name="s3",
             aws_access_key_id=os.environ.get("UPLOAD_ASSETS_ACCESS_KEY_ID"),
@@ -59,6 +79,19 @@ def pytest_configure(config: pytest.Config):
         )
 
 
+def pytest_report_header(config):
+    plugin: S3UploadPlugin | None = config.pluginmanager.get_plugin(
+        _UPLOAD_ASSETS_PLUGIN_NAME
+    )
+    if plugin:
+        return f"Plugin `upload_assets` is active, with upload to {plugin.bucket!r}"
+
+
+def pytest_unconfigure(config):
+    if config.pluginmanager.hasplugin(_UPLOAD_ASSETS_PLUGIN_NAME):
+        config.pluginmanager.unregister(name=_UPLOAD_ASSETS_PLUGIN_NAME)
+
+
 class _Collector:
     """
     Collects test outcomes and files to upload for a single test node.
@@ -112,17 +145,29 @@ def _upload(self, collector: _Collector):
 
 
 @pytest.fixture
-def upload_assets(pytestconfig, tmp_path) -> Callable[[Path], None]:
+def upload_assets(pytestconfig: pytest.Config, tmp_path) -> Callable:
     """
     Fixture to register a file (under `tmp_path`) for S3 upload
-    after the test failed.
+    after the test failed. The fixture is a function that
+    can be called with one or more `Path` objects to upload.
     """
-    uploader = pytestconfig.pluginmanager.get_plugin(_UPLOAD_ASSETS_PLUGIN_NAME)
+    uploader: S3UploadPlugin | None = pytestconfig.pluginmanager.get_plugin(
+        _UPLOAD_ASSETS_PLUGIN_NAME
+    )
+
+    if uploader:
+
+        def collect(*paths: Path):
+            for path in paths:
+                # TODO: option to make relative from other root
+                #       (e.g. when test uses an `actual` folder for actual results)
+                assert path.is_relative_to(tmp_path)
+                name = str(path.relative_to(tmp_path))
+                uploader.collector.collect(path=path, name=name)
+    else:
+        warnings.warn("Fixture `upload_assets` is a no-op (incomplete set up).")
 
-    def collect(*paths: Path):
-        for path in paths:
-            assert path.is_relative_to(tmp_path)
-            name = str(path.relative_to(tmp_path))
-            uploader.collector.collect(path=path, name=name)
+        def collect(*paths: Path):
+            pass
 
     return collect