feat(alert): Add collector fail alert

canonical · Nov 23, 2023 · 91128dc · 91128dc
1 parent 7069721
commit 91128dc
Show file tree

Hide file tree

Showing 2 changed files with 48 additions and 0 deletions.
diff --git a/src/prometheus_alert_rules/general.yaml b/src/prometheus_alert_rules/general.yaml
@@ -0,0 +1,13 @@
+groups:
+- name: HardwareObserver
+  rules:
+    - alert: CollectorFailed
+      expr: '{__name__=~"(.*)_collector_failed"} == 1'
+      for: 0m
+      labels:
+        severity: warning
+      annotations:
+        summary: Controller fetch failed. (instance {{ $labels.instance }})
+        description: |
+          Collrector fetch failed. Please reach out to hardware-observer maintainers.
+            LABELS = {{ $labels }}
diff --git a/tests/unit/test_alert_rules/test_general.yaml b/tests/unit/test_alert_rules/test_general.yaml
@@ -0,0 +1,35 @@
+rule_files:
+  - ../../../src/prometheus_alert_rules/general.yaml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: ipmidcmicollector_collector_failed{instance="ubuntu-99", collector="IPMIDCMICollector"}
+        values: '1x15'
+      - series: ipmiselcollector_collector_failed{instance="ubuntu-99", collector="IPMISELCollector"}
+        values: '1x15'
+
+    alert_rule_test:
+      - eval_time: 0m
+        alertname: CollectorFailed
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+              instance: ubuntu-99
+              collector: IPMIDCMICollector
+            exp_annotations:
+              summary: Controller fetch failed. (instance ubuntu-99)
+              description: |
+                Collrector fetch failed. Please reach out to hardware-observer maintainers.
+                  LABELS = map[__name__:ipmidcmicollector_collector_failed collector:IPMIDCMICollector instance:ubuntu-99]
+          - exp_labels:
+              severity: warning
+              instance: ubuntu-99
+              collector: IPMISELCollector
+            exp_annotations:
+              summary: Controller fetch failed. (instance ubuntu-99)
+              description: |
+                Collrector fetch failed. Please reach out to hardware-observer maintainers.
+                  LABELS = map[__name__:ipmiselcollector_collector_failed collector:IPMISELCollector instance:ubuntu-99]