feat(kps): move to alertmanager spec

buroa · Dec 11, 2024 · 26295c0 · 26295c0
1 parent 41e9634
commit 26295c0
Show file tree

Hide file tree

Showing 7 changed files with 115 additions and 87 deletions.
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
@@ -32,8 +32,10 @@ spec:
         ingressClassName: internal
         hosts: ["am.ktwo.io"]
       alertmanagerSpec:
-        useExistingSecret: true
-        configSecret: alertmanager-secret
+        alertmanagerConfiguration:
+          name: alertmanager
+          global:
+            resolveTimeout: 5m
         externalUrl: https://am.ktwo.io
         storage:
           volumeClaimTemplate:

diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml
@@ -2,11 +2,4 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ./externalsecret.yaml
   - ./helmrelease.yaml
-configMapGenerator:
-  - name: alertmanager-config-tpl
-    files:
-      - ./resources/alertmanager.yaml
-generatorOptions:
-  disableNameSuffixHash: true
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml
@@ -0,0 +1,77 @@
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: alertmanager
+spec:
+  route:
+    groupBy: ["alertname", "job"]
+    groupInterval: 10m
+    groupWait: 1m
+    receiver: pushover
+    repeatInterval: 12h
+    routes:
+      - receiver: "null"
+        matchers: [{name: alertname, value: InfoInhibitor, matchType: =}]
+      - receiver: heartbeat
+        groupInterval: 15s
+        groupWait: 0s
+        repeatInterval: 5m
+        matchers: [{name: alertname, value: Watchdog, matchType: =}]
+      - receiver: pushover
+        matchers: [{name: severity, value: critical, matchType: =}]
+
+  inhibitRules:
+    - equal: ["alertname", "namespace"]
+      sourceMatch:
+        - name: severity
+          value: critical
+          matchType: =
+        - name: severity
+          value: warning
+          matchType: =~
+
+  receivers:
+    - name: "null"
+    - name: heartbeat
+      webhookConfigs:
+        - urlSecret:
+            name: &secret alertmanager-secret
+            key: ALERTMANAGER_HEARTBEAT_URL
+    - name: pushover
+      pushoverConfigs:
+        - html: true
+          message: |-
+            {{- range .Alerts }}
+              {{- if ne .Annotations.description "" }}
+                {{ .Annotations.description }}
+              {{- else if ne .Annotations.summary "" }}
+                {{ .Annotations.summary }}
+              {{- else if ne .Annotations.message "" }}
+                {{ .Annotations.message }}
+              {{- else }}
+                Alert description not available
+              {{- end }}
+              {{- if gt (len .Labels.SortedPairs) 0 }}
+                <small>
+                {{- range .Labels.SortedPairs }}
+                  <b>{{ .Name }}:</b> {{ .Value }}
+                {{- end }}
+                </small>
+              {{- end }}
+            {{- end }}
+          priority: |-
+            {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+          sendResolved: true
+          sound: gamelan
+          title: >-
+            [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
+            {{ .CommonLabels.alertname }}
+          ttl: 3600s
+          token:
+            name: *secret
+            key: ALERTMANAGER_PUSHOVER_APP_TOKEN
+          userKey:
+            name: *secret
+            key: ALERTMANAGER_PUSHOVER_USER_KEY
+          urlTitle: View in Alertmanager
diff --git a/...-prometheus-stack/app/externalsecret.yaml → ...metheus-stack/configs/externalsecret.yaml b/...-prometheus-stack/app/externalsecret.yaml → ...metheus-stack/configs/externalsecret.yaml
@@ -4,19 +4,18 @@ kind: ExternalSecret
 metadata:
   name: alertmanager
 spec:
-  refreshInterval: 5m
   secretStoreRef:
     kind: ClusterSecretStore
     name: onepassword-connect
   target:
     name: alertmanager-secret
     creationPolicy: Owner
     template:
-      templateFrom:
-        - configMap:
-            name: alertmanager-config-tpl
-            items:
-              - key: alertmanager.yaml
+      engineVersion: v2
+      data:
+        ALERTMANAGER_HEARTBEAT_URL: "{{ .ALERTMANAGER_HEARTBEAT_URL }}"
+        ALERTMANAGER_PUSHOVER_APP_TOKEN: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}"
+        ALERTMANAGER_PUSHOVER_USER_KEY: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}"
   dataFrom:
     - extract:
         key: alertmanager
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./alertmanagerconfig.yaml
+  - ./externalsecret.yaml
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml
@@ -10,7 +10,6 @@ spec:
     labels:
       app.kubernetes.io/name: *app
   dependsOn:
-    - name: external-secrets-stores
     - name: rook-ceph-cluster
   path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app
   prune: true
@@ -24,6 +23,29 @@ spec:
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
+metadata:
+  name: &app kube-prometheus-stack-configs
+  namespace: flux-system
+spec:
+  targetNamespace: monitoring
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: external-secrets-stores
+    - name: kube-prometheus-stack
+  path: ./kubernetes/apps/monitoring/kube-prometheus-stack/configs
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: k8s-gitops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 15m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
 metadata:
   name: &app kube-prometheus-stack-tools
   namespace: flux-system