fix: General err handling for collector

prometheus_hardware_exporter/core.py

@@ -5,7 +5,7 @@
 from logging import getLogger
 from typing import Any, Dict, Iterable, List, Type
 
-from prometheus_client.metrics_core import Metric
+from prometheus_client.metrics_core import GaugeMetricFamily, Metric
 from prometheus_client.registry import Collector
 
 from .config import Config
@@ -89,6 +89,25 @@ def specifications(self) -> List[Specification]:
             A list of specification.
         """
 
+    @property
+    def failed_metrics(self) -> Iterable[Metric]:
+        """Defines the metrics to be returned when collector fails.
+
+        Yields:
+            metrics: the internal metrics
+        """
+        name = self.__class__.__name__
+        metric = GaugeMetricFamily(
+            name=f"{name.lower()}_collector_failed",
+            documentation=f"{name} Collector failed to fetch metrics",
+            labels=["collector"],
+        )
+        metric.add_metric(
+            labels=[self.__class__.__name__],
+            value=1,
+        )
+        yield metric
+
     def init_default_datastore(self, payloads: List[Payload]) -> None:
         """Initialize or fill data the store with default values.
 
@@ -111,21 +130,28 @@ def collect(self) -> Iterable[Metric]:
         Yields:
             metrics: the internal metrics
         """
-        payloads = self.fetch()
-        self.init_default_datastore(payloads)
-        processed_payloads = self.process(payloads, self._datastore)
-
-        # unpacked and create metrics
-        for payload in processed_payloads:
-            spec = self._specs[payload.name]
-            # We have to ignore the type checking here, since the subclass of
-            # any metric family from prometheus client adds new attributes and
-            # methods.
-            metric = spec.metric_class(  # type: ignore[call-arg]
-                name=spec.name, labels=spec.labels, documentation=spec.documentation
-            )
-            metric.add_metric(  # type: ignore[attr-defined]
-                labels=payload.labels, value=payload.value
-            )
-            yield metric
-            self._datastore[payload.uuid] = payload
+        # The general exception hanlder will try to make sure the single
+        # collector's bug will only change the metrics output to failed_metrics
+        # and also make sure other collectors are still working.
+        try:
+            payloads = self.fetch()
+            self.init_default_datastore(payloads)
+            processed_payloads = self.process(payloads, self._datastore)
+
+            # unpacked and create metrics
+            for payload in processed_payloads:
+                spec = self._specs[payload.name]
+                # We have to ignore the type checking here, since the subclass of
+                # any metric family from prometheus client adds new attributes and
+                # methods.
+                metric = spec.metric_class(  # type: ignore[call-arg]
+                    name=spec.name, labels=spec.labels, documentation=spec.documentation
+                )
+                metric.add_metric(  # type: ignore[attr-defined]
+                    labels=payload.labels, value=payload.value
+                )
+                yield metric
+                self._datastore[payload.uuid] = payload
+        except Exception as err:  # pylint: disable=W0718
+            logger.error(err)
+            yield from self.failed_metrics

tests/unit/test_collector.py

@@ -1235,3 +1235,31 @@ def test_210_redfish_create_smart_storage_health_metric_payload(self):
                 )
             ],
         )
+
+    def test_1000_collector_fetch_failed(self):
+        for collector_cls, expected_name, expected_labels in [
+            (
+                MegaRAIDCollector,
+                "megaraidcollector_collector_failed",
+                {"collector": "MegaRAIDCollector"},
+            ),
+            (
+                RedfishCollector,
+                "redfishcollector_collector_failed",
+                {"collector": "RedfishCollector"},
+            ),
+            (
+                IpmiSensorsCollector,
+                "ipmisensorscollector_collector_failed",
+                {"collector": "IpmiSensorsCollector"},
+            ),
+        ]:
+            collector = collector_cls(Mock())
+            collector.fetch = Mock()
+            collector.fetch.side_effect = Exception("Unknown error")
+            payloads = collector.collect()
+            payloads = list(payloads)
+            self.assertEqual(len(payloads), 1)
+            self.assertEqual(payloads[0].name, expected_name)
+            self.assertEqual(payloads[0].samples[0].value, 1.0)
+            self.assertEqual(payloads[0].samples[0].labels, expected_labels)