feat: INFRA-685 fix minio disk alerts

Ferlab-Ste-Justine · Sep 8, 2024 · d9874f4 · d9874f4
1 parent 51a9a2c
commit d9874f4
Showing 1 changed file with 18 additions and 10 deletions.
diff --git a/templates/node-exporter.yml.tpl b/templates/node-exporter.yml.tpl
@@ -56,13 +56,8 @@ groups:
         expr: count without (device, major, minor, serial, path, model, revision) (node_disk_info{device=~"sd.|vd.",job="${job.tag}-node-exporter"})
 %{ if job.expected_disks_count >= 0 ~}
       - alert: ${replace(title(replace(job.tag, "-", " ")), " ", "")}DiskCountMismatch
-        expr: (${replace(job.tag, "-", "_")}:disks:count != ${job.expected_disks_count})
+        expr: ${replace(job.tag, "-", "_")}:disks:count != ${job.expected_disks_count}
         for: 15m
-%{ elif job.min_disks_count >= 0 and job.max_disks_count >= 0 ~}
-      - alert: ${replace(title(replace(job.tag, "-", " ")), " ", "")}DiskCountMismatch
-        expr: (${replace(job.tag, "-", "_")}:disks:count < ${job.min_disks_count} or ${replace(job.tag, "-", "_")}:disks:count > ${job.max_disks_count})
-        for: 15m
-%{ endif ~}
 %{ if length(job.alert_labels) > 0 ~}
         labels:
 %{ for key, val in job.alert_labels ~}
@@ -71,10 +66,23 @@ groups:
 %{ endif ~}
         annotations:
           summary: "${title(replace(job.tag, "-", " "))} Number of Disks Unexpected"
-%{ if job.expected_disks_count >= 0 ~}
-          description: "Instance *{{ $labels.instance }}* of job *{{ $labels.job }}* has *{{ $value }}* disks. Expected *${job.expected_disks_count}* disks."
-%{ elif job.min_disks_count >= 0 and job.max_disks_count >= 0 ~}
-          description: "Instance *{{ $labels.instance }}* of job *{{ $labels.job }}* has *{{ $value }}* disks. Expected between *${job.min_disks_count}* and *${job.max_disks_count}* disks."
+          description: "Instance *{{ $labels.instance }}* of job *{{ $labels.job }}* has *{{ $value }}* disks. Expected *${job.expected_disks_count}*."
+%{ else ~}
+%{ if job.min_disks_count >= 0 ~}
+  %{ if job.max_disks_count >= 0 }
+    - alert: ${replace(title(replace(job.tag, "-", " ")), " ", "")}DiskCountRangeMismatch
+      expr: (${replace(job.tag, "-", "_")}:disks:count < ${job.min_disks_count} or ${replace(job.tag, "-", "_")}:disks:count > ${job.max_disks_count})
+      for: 15m
+%{ if length(job.alert_labels) > 0 ~}
+      labels:
+%{ for key, val in job.alert_labels ~}
+        ${key}: "${val}"
+%{ endfor ~}
+%{ endif ~}
+      annotations:
+        summary: "${title(replace(job.tag, "-", " "))} Disk Count Out of Range"
+        description: "Instance *{{ $labels.instance }}* of job *{{ $labels.job }}* has *{{ $value }}* disks. Expected between *${job.min_disks_count}* and *${job.max_disks_count}*."
+%{ endif }
 %{ endif ~}
 %{ endif ~}
       - record: ${replace(job.tag, "-", "_")}:filesystem_size:gigabytes