fix(alerts): Increase duration before redfish alerts get triggered

Increase the duration before which redfish alerts get triggered. This prevents false alerts during ocassional flapping.
canonical · Mar 21, 2024 · 70e6cdd · 70e6cdd
1 parent 7523866
commit 70e6cdd
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 18 deletions.
diff --git a/src/prometheus_alert_rules/redfish.yaml b/src/prometheus_alert_rules/redfish.yaml
@@ -3,7 +3,7 @@ groups:
     rules:
       - alert: RedfishCallFailed
         expr: redfish_call_success == 0
-        for: 0m
+        for: 5m
         labels:
           severity: warning
         annotations:
@@ -15,7 +15,7 @@ groups:
 
       - alert: RedfishServiceUnavailable
         expr: redfish_service_available == 0
-        for: 0m
+        for: 5m
         labels:
           severity: warning
         annotations:
@@ -27,7 +27,7 @@ groups:
 
       - alert: RedfishSensorHealthNotOk
         expr: redfish_sensor_info{health!~"OK|N/A"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -39,7 +39,7 @@ groups:
 
       - alert: RedfishProcessorHealthNotOk
         expr: redfish_processor_info{health!~"OK|NA"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -50,7 +50,7 @@ groups:
 
       - alert: RedfishStorageControllerHealthNotOk
         expr: redfish_storage_controller_info{health!~"OK|NA"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -61,7 +61,7 @@ groups:
 
       - alert: RedfishChassisHealthNotOk
         expr: redfish_chassis_info{health!~"OK|NA"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -72,7 +72,7 @@ groups:
 
       - alert: RedfishStorageDriveHealthNotOk
         expr: redfish_storage_drive_info{health!~"OK|NA"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -83,7 +83,7 @@ groups:
 
       - alert: RedfishMemoryDimmHealthNotOk
         expr: redfish_memory_dimm_info{health!~"OK|NA"}
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:
@@ -94,7 +94,7 @@ groups:
 
       - alert: RedfishSmartStorageHealthNotOk
         expr: redfish_smart_storage_health == 0
-        for: 0m
+        for: 5m
         labels:
           severity: critical
         annotations:

diff --git a/tests/unit/test_alert_rules/test_redfish.yaml b/tests/unit/test_alert_rules/test_redfish.yaml
@@ -10,7 +10,7 @@ tests:
         values: "0x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishCallFailed
         exp_alerts:
           - exp_labels:
@@ -29,7 +29,7 @@ tests:
         values: "0x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishServiceUnavailable
         exp_alerts:
           - exp_labels:
@@ -48,7 +48,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishSensorHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -69,7 +69,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishProcessorHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -91,7 +91,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishStorageControllerHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -113,7 +113,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishChassisHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -134,7 +134,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishStorageDriveHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -156,7 +156,7 @@ tests:
         values: "1x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishMemoryDimmHealthNotOk
         exp_alerts:
           - exp_labels:
@@ -177,7 +177,7 @@ tests:
         values: "0x15"
 
     alert_rule_test:
-      - eval_time: 0m
+      - eval_time: 10m
         alertname: RedfishSmartStorageHealthNotOk
         exp_alerts:
           - exp_labels: