diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
index 7550251b65..c21a66c0e4 100644
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
@@ -32,8 +32,10 @@ spec:
ingressClassName: internal
hosts: ["am.ktwo.io"]
alertmanagerSpec:
- useExistingSecret: true
- configSecret: alertmanager-secret
+ alertmanagerConfiguration:
+ name: alertmanager
+ global:
+ resolveTimeout: 5m
externalUrl: https://am.ktwo.io
storage:
volumeClaimTemplate:
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml
index 004b793183..5dd7baca73 100644
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml
@@ -2,11 +2,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- - ./externalsecret.yaml
- ./helmrelease.yaml
-configMapGenerator:
- - name: alertmanager-config-tpl
- files:
- - ./resources/alertmanager.yaml
-generatorOptions:
- disableNameSuffixHash: true
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml
deleted file mode 100644
index fd5990ce5a..0000000000
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
----
-global:
- resolve_timeout: 5m
-
-route:
- group_by: ["alertname", "job"]
- group_interval: 10m
- group_wait: 1m
- receiver: pushover
- repeat_interval: 12h
- routes:
- - receiver: heartbeat
- group_interval: 15s
- group_wait: 0s
- repeat_interval: 5m
- matchers:
- - alertname =~ "Watchdog"
- - receiver: "null"
- matchers:
- - alertname =~ "InfoInhibitor"
- - receiver: pushover
- continue: true
- matchers:
- - severity = "critical"
-
-inhibit_rules:
- - equal: ["alertname", "namespace"]
- source_matchers:
- - severity = "critical"
- target_matchers:
- - severity = "warning"
-
-receivers:
- - name: heartbeat
- webhook_configs:
- - send_resolved: true
- url: "{{ .ALERTMANAGER_HEARTBEAT_URL }}"
- - name: "null"
- - name: pushover
- pushover_configs:
- - html: true
- message: |-
- {{ "{{-" }} range .Alerts {{ "}}" }}
- {{ "{{-" }} if ne .Annotations.description "" {{ "}}" }}
- {{ "{{" }} .Annotations.description {{ "}}" }}
- {{ "{{-" }} else if ne .Annotations.summary "" {{ "}}" }}
- {{ "{{" }} .Annotations.summary {{ "}}" }}
- {{ "{{-" }} else if ne .Annotations.message "" {{ "}}" }}
- {{ "{{" }} .Annotations.message {{ "}}" }}
- {{ "{{-" }} else {{ "}}" }}
- Alert description not available
- {{ "{{-" }} end {{ "}}" }}
- {{ "{{-" }} if gt (len .Labels.SortedPairs) 0 {{ "}}" }}
-
- {{ "{{-" }} range .Labels.SortedPairs {{ "}}" }}
- {{ "{{" }} .Name {{ "}}" }}: {{ "{{" }} .Value {{ "}}" }}
- {{ "{{-" }} end {{ "}}" }}
-
- {{ "{{-" }} end {{ "}}" }}
- {{ "{{-" }} end {{ "}}" }}
- priority: |-
- {{ "{{" }} if eq .Status "firing" {{ "}}" }}1{{ "{{" }} else {{ "}}" }}0{{ "{{" }} end {{ "}}" }}
- send_resolved: true
- sound: gamelan
- title: >-
- [{{ "{{" }} .Status | toUpper {{ "}}" }}{{ "{{" }} if eq .Status "firing" {{ "}}" }}:{{ "{{" }} .Alerts.Firing | len {{ "}}" }}{{ "{{" }} end {{ "}}" }}]
- {{ "{{" }} .CommonLabels.alertname {{ "}}" }}
- token: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}"
- # ttl: "{{ .ALERTMANAGER_PUSHOVER_TTL }}"
- url_title: View in Alertmanager
- user_key: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}"
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml
new file mode 100644
index 0000000000..b06279be8f
--- /dev/null
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml
@@ -0,0 +1,77 @@
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+ name: alertmanager
+spec:
+ route:
+ groupBy: ["alertname", "job"]
+ groupInterval: 10m
+ groupWait: 1m
+ receiver: pushover
+ repeatInterval: 12h
+ routes:
+ - receiver: "null"
+ matchers: [{name: alertname, value: InfoInhibitor, matchType: =}]
+ - receiver: heartbeat
+ groupInterval: 15s
+ groupWait: 0s
+ repeatInterval: 5m
+ matchers: [{name: alertname, value: Watchdog, matchType: =}]
+ - receiver: pushover
+ matchers: [{name: severity, value: critical, matchType: =}]
+
+ inhibitRules:
+ - equal: ["alertname", "namespace"]
+ sourceMatch:
+ - name: severity
+ value: critical
+ matchType: =
+ - name: severity
+ value: warning
+ matchType: =~
+
+ receivers:
+ - name: "null"
+ - name: heartbeat
+ webhookConfigs:
+ - urlSecret:
+ name: &secret alertmanager-secret
+ key: ALERTMANAGER_HEARTBEAT_URL
+ - name: pushover
+ pushoverConfigs:
+ - html: true
+ message: |-
+ {{- range .Alerts }}
+ {{- if ne .Annotations.description "" }}
+ {{ .Annotations.description }}
+ {{- else if ne .Annotations.summary "" }}
+ {{ .Annotations.summary }}
+ {{- else if ne .Annotations.message "" }}
+ {{ .Annotations.message }}
+ {{- else }}
+ Alert description not available
+ {{- end }}
+ {{- if gt (len .Labels.SortedPairs) 0 }}
+
+ {{- range .Labels.SortedPairs }}
+ {{ .Name }}: {{ .Value }}
+ {{- end }}
+
+ {{- end }}
+ {{- end }}
+ priority: |-
+ {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+ sendResolved: true
+ sound: gamelan
+ title: >-
+ [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
+ {{ .CommonLabels.alertname }}
+ ttl: 3600s
+ token:
+ name: *secret
+ key: ALERTMANAGER_PUSHOVER_APP_TOKEN
+ userKey:
+ name: *secret
+ key: ALERTMANAGER_PUSHOVER_USER_KEY
+ urlTitle: View in Alertmanager
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml
similarity index 53%
rename from kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml
rename to kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml
index 5b44b3c860..c431723ba0 100644
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml
@@ -4,7 +4,6 @@ kind: ExternalSecret
metadata:
name: alertmanager
spec:
- refreshInterval: 5m
secretStoreRef:
kind: ClusterSecretStore
name: onepassword-connect
@@ -12,11 +11,11 @@ spec:
name: alertmanager-secret
creationPolicy: Owner
template:
- templateFrom:
- - configMap:
- name: alertmanager-config-tpl
- items:
- - key: alertmanager.yaml
+ engineVersion: v2
+ data:
+ ALERTMANAGER_HEARTBEAT_URL: "{{ .ALERTMANAGER_HEARTBEAT_URL }}"
+ ALERTMANAGER_PUSHOVER_APP_TOKEN: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}"
+ ALERTMANAGER_PUSHOVER_USER_KEY: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}"
dataFrom:
- extract:
key: alertmanager
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml
new file mode 100644
index 0000000000..853ffc1d4f
--- /dev/null
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml
@@ -0,0 +1,6 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+ - ./alertmanagerconfig.yaml
+ - ./externalsecret.yaml
diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml
index bcaae4b300..ecdf9263f4 100644
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml
@@ -10,7 +10,6 @@ spec:
labels:
app.kubernetes.io/name: *app
dependsOn:
- - name: external-secrets-stores
- name: rook-ceph-cluster
path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app
prune: true
@@ -24,6 +23,29 @@ spec:
---
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
+metadata:
+ name: &app kube-prometheus-stack-configs
+ namespace: flux-system
+spec:
+ targetNamespace: monitoring
+ commonMetadata:
+ labels:
+ app.kubernetes.io/name: *app
+ dependsOn:
+ - name: external-secrets-stores
+ - name: kube-prometheus-stack
+ path: ./kubernetes/apps/monitoring/kube-prometheus-stack/configs
+ prune: true
+ sourceRef:
+ kind: GitRepository
+ name: k8s-gitops
+ wait: true
+ interval: 30m
+ retryInterval: 1m
+ timeout: 15m
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
metadata:
name: &app kube-prometheus-stack-tools
namespace: flux-system