tests: Modify deployment for real hw tests

This patch modifies the test_build_and_deploy method to work even when
testing with real hardware where the machine has already been
pre-deployed in the model. The attaching of required resources after the
bundle deployment is handled here as well.

The test execution logic has also been slightly tweaked. If the
--collectors option is provided, then ONLY the real hardware dependent
tests will be executed (marked with the "realhw" marker, belonging to
TestCharmWithHW). If the option not provided, the hardware independent
tests (in TestCharm) will be executed. If full tests are required, a
make target can be created which runs the pytest command with and without
providing collectors.

Some logic has also been added to ensure that the test_build_and_deploy
function will run for both, hardware dependent and independent tests.

tests/functional/bundle.yaml.j2

@@ -16,7 +16,7 @@ applications:
       - "0"
   grafana-agent:
     charm: grafana-agent
-    channel: edge # FIXME: currently no stable release
+    channel: stable
   hardware-observer:
     charm: {{ charm }}
 

tests/functional/conftest.py

@@ -1,6 +1,9 @@
 import logging
 
 import pytest
+from utils import RESOURCES_DIR, Resource
+
+from config import EXPORTER_COLLECTOR_MAPPING, TPR_RESOURCES, HWTool
 
 log = logging.getLogger(__name__)
 
@@ -49,14 +52,24 @@ def pytest_configure(config):
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("collectors"):
-        # --collectors provided, do not skip tests
-        return
-    skip_real_hw = pytest.mark.skip(
-        reason="Hardware dependent test. Provide collectors with the --collectors option."
-    )
-    for item in items:
-        if "realhw" in item.keywords:
-            item.add_marker(skip_real_hw)
+        # --collectors provided, skip hw independent tests
+        skip_hw_independent = pytest.mark.skip(
+            reason="Hardware independent tests are skipped since --collectors was provided."
+        )
+        for item in items:
+            # skip TestCharm tests where "realhw" marker is not present
+            # we don't want to skip test_setup_and_build even for hw independent tests
+            # so we also check for the skip_if_deployed marker
+            if "realhw" not in item.keywords and "skip_if_deployed" not in item.keywords:
+                item.add_marker(skip_hw_independent)
+    else:
+        # skip hw dependent tests in TestCharmWithHW marked with "realhw"
+        skip_hw_dependent = pytest.mark.skip(
+            reason="Hardware dependent test. Provide collectors with the --collectors option."
+        )
+        for item in items:
+            if "realhw" in item.keywords:
+                item.add_marker(skip_hw_dependent)
 
 
 @pytest.fixture()
@@ -67,3 +80,61 @@ def app(ops_test):
 @pytest.fixture()
 def unit(app):
     return app.units[0]
+
+
+@pytest.fixture()
+def resources() -> list[Resource]:
+    """Return list of Resource objects."""
+    return [
+        Resource(
+            resource_name=TPR_RESOURCES.get(HWTool.STORCLI),
+            file_name="storcli.deb",
+            collector_name=EXPORTER_COLLECTOR_MAPPING.get(HWTool.STORCLI)[0].replace(
+                "collector.", ""
+            ),
+            bin_name=HWTool.STORCLI.value,
+        ),
+        Resource(
+            resource_name=TPR_RESOURCES.get(HWTool.PERCCLI),
+            file_name="perccli.deb",
+            collector_name=EXPORTER_COLLECTOR_MAPPING.get(HWTool.PERCCLI)[0].replace(
+                "collector.", ""
+            ),
+            bin_name=HWTool.PERCCLI.value,
+        ),
+        Resource(
+            resource_name=TPR_RESOURCES.get(HWTool.SAS2IRCU),
+            file_name="sas2ircu",
+            collector_name=EXPORTER_COLLECTOR_MAPPING.get(HWTool.SAS2IRCU)[0].replace(
+                "collector.", ""
+            ),
+            bin_name=HWTool.SAS2IRCU.value,
+        ),
+        Resource(
+            resource_name=TPR_RESOURCES.get(HWTool.SAS3IRCU),
+            file_name="sas3ircu",
+            collector_name=EXPORTER_COLLECTOR_MAPPING.get(HWTool.SAS3IRCU)[0].replace(
+                "collector.", ""
+            ),
+            bin_name=HWTool.SAS3IRCU.value,
+        ),
+    ]
+
+
+@pytest.fixture()
+def required_resources(resources: list[Resource], provided_collectors: set) -> list[Resource]:
+    """Return list of required resources to be attached as per hardware availability.
+
+    Required resources will be empty if no collectors are provided.
+    """
+    collector_names = [r.collector_name for r in resources]
+    required_resources = []
+
+    for resource, collector_name in zip(resources, collector_names):
+        if collector_name in provided_collectors:
+            required_resources.append(resource)
+
+    for resource in required_resources:
+        resource.file_path = f"{RESOURCES_DIR}/{resource.file_name}"
+
+    return required_resources

tests/functional/test_charm.py

@@ -12,7 +12,7 @@
 import pytest
 import yaml
 from pytest_operator.plugin import OpsTest
-from utils import get_metrics_output, parse_metrics, run_command_on_unit
+from utils import RESOURCES_DIR, get_metrics_output, parse_metrics, run_command_on_unit
 
 logger = logging.getLogger(__name__)
 
@@ -43,10 +43,13 @@ class AppStatus(str, Enum):
 
 @pytest.mark.abort_on_fail
 @pytest.mark.skip_if_deployed
-async def test_build_and_deploy(ops_test: OpsTest, series):
+async def test_build_and_deploy(  # noqa: C901, function is too complex
+    ops_test: OpsTest, series, provided_collectors, required_resources
+):
     """Build the charm-under-test and deploy it together with related charms.
 
     Assert on the unit status before any relations/configurations take place.
+    Optionally attach required resources when testing with real hardware.
     """
     # Build and deploy charm from local source folder
     charm = await ops_test.build_charm(".")
@@ -66,8 +69,15 @@ async def test_build_and_deploy(ops_test: OpsTest, series):
             "sas3ircu-bin": "empty-resource",
         },
     )
+
     juju_cmd = ["deploy", "-m", ops_test.model_full_name, str(bundle)]
 
+    # deploy bundle to already added machine instead of provisioning new one
+    # when testing with real hardware
+    if provided_collectors:
+        juju_cmd.append("--map-machines=existing")
+
+    logging.info("Deploying bundle...")
     rc, stdout, stderr = await ops_test.juju(*juju_cmd)
     assert rc == 0, f"Bundle deploy failed: {(stderr or stdout).strip()}"
 
@@ -88,7 +98,42 @@ async def test_build_and_deploy(ops_test: OpsTest, series):
         timeout=TIMEOUT,
     )
 
-    # Test initial workload status
+    logging.info(
+        f"Required resources to attach with charm: {[r.resource_name for r in required_resources]}"
+    )
+
+    if required_resources:
+        # check workload status for real hardware based tests requiring resources to be attached
+        for unit in ops_test.model.applications[APP_NAME].units:
+            assert AppStatus.MISSING_RESOURCES in unit.workload_status_message
+
+        # NOTE: resource files need to be manually placed into the resources directory
+        for resource in required_resources:
+            path = f"{RESOURCES_DIR}/{resource.file_name}"
+            if not Path(path).exists():
+                pytest.fail(f"{path} not provided. Add resource into {RESOURCES_DIR} directory")
+            resource.file_path = path
+
+        resource_path_map = {r.resource_name: r.file_path for r in required_resources}
+        resource_cmd = " ".join(
+            f"{resource_name}={resource_path}"
+            for resource_name, resource_path in resource_path_map.items()
+        )
+        juju_cmd = ["attach-resource", APP_NAME, "-m", ops_test.model_full_name, resource_cmd]
+
+        logging.info("Attaching resources...")
+        rc, stdout, stderr = await ops_test.juju(*juju_cmd)
+        assert rc == 0, f"Attaching resources failed: {(stderr or stdout).strip()}"
+
+        # still blocked since cos-agent relation has not been added
+        await ops_test.model.wait_for_idle(
+            apps=[APP_NAME],
+            status="blocked",
+            timeout=TIMEOUT,
+        )
+
+    # Test workload status with no real hardware or
+    # when real hardware doesn't require any resource to be attached
     for unit in ops_test.model.applications[APP_NAME].units:
         assert AppStatus.MISSING_RESOURCES not in unit.workload_status_message
         assert unit.workload_status_message == AppStatus.MISSING_RELATION
@@ -101,12 +146,14 @@ async def test_build_and_deploy(ops_test: OpsTest, series):
     check_active_cmd = "systemctl is-active hardware-exporter"
 
     # Test without cos-agent relation
+    logging.info("Check whether hardware-exporter is inactive before creating relation.")
     for unit in ops_test.model.applications[APP_NAME].units:
         results = await run_command_on_unit(ops_test, unit.name, check_active_cmd)
         assert results.get("return-code") > 0
         assert results.get("stdout").strip() == "inactive"
 
     # Add cos-agent relation
+    logging.info("Adding cos-agent relation.")
     await asyncio.gather(
         ops_test.model.add_relation(
             f"{APP_NAME}:cos-agent", f"{GRAFANA_AGENT_APP_NAME}:cos-agent"
@@ -119,6 +166,7 @@ async def test_build_and_deploy(ops_test: OpsTest, series):
     )
 
     # Test with cos-agent relation
+    logging.info("Check whether hardware-exporter is active after creating relation.")
     for unit in ops_test.model.applications[APP_NAME].units:
         results = await run_command_on_unit(ops_test, unit.name, check_active_cmd)
         assert results.get("return-code") == 0

tests/functional/utils.py

@@ -3,12 +3,42 @@
 import re
 from collections import defaultdict
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Optional
 
 from async_lru import alru_cache
 
 from config import EXPORTER_DEFAULT_PORT
 
+RESOURCES_DIR = Path("./resources/")
+
+
+@dataclass
+class Metric:
+    """Class for metric data."""
+
+    name: str
+    labels: Optional[str]
+    value: float
+
+
+@dataclass
+class Resource:
+    """Class for resource data.
+
+    resource_name: Name of juju resource for charm
+    file_name: file name for resource
+    collector_name: Associated collector name for resource
+    bin_name: Name of the binary after installing resource
+    file_path: Path to resource file to be attached (None by default)
+    """
+
+    resource_name: str
+    file_name: str
+    collector_name: str
+    bin_name: str
+    file_path: Optional[str] = None
+
 
 async def run_command_on_unit(ops_test, unit_name, command):
     complete_command = ["exec", "--unit", unit_name, "--", *command.split()]
@@ -28,15 +58,6 @@ async def get_metrics_output(ops_test, unit_name):
     return results
 
 
-@dataclass
-class Metric:
-    """Class for metric data."""
-
-    name: str
-    labels: Optional[str]
-    value: float
-
-
 def _parse_single_metric(metric: str) -> Optional[Metric]:
     """Return a Metric object parsed from a single metric string."""
     # ignore blank lines or comments