feat(kps): move to alertmanager spec

kubernetes/apps/monitoring/karma/ks.yaml

@@ -19,4 +19,4 @@ spec:
   wait: true
   interval: 30m
   retryInterval: 1m
-  timeout: 15m
+  timeout: 5m

kubernetes/apps/monitoring/kube-prometheus-stack/app/alertmanagerconfig.yaml

@@ -0,0 +1,77 @@
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: alertmanager
+spec:
+  route:
+    groupBy: ["alertname", "job"]
+    groupInterval: 10m
+    groupWait: 1m
+    receiver: pushover
+    repeatInterval: 12h
+    routes:
+      - receiver: "null"
+        matchers: [{name: alertname, value: InfoInhibitor, matchType: =}]
+      - receiver: heartbeat
+        groupInterval: 15s
+        groupWait: 0s
+        repeatInterval: 5m
+        matchers: [{name: alertname, value: Watchdog, matchType: =}]
+      - receiver: pushover
+        matchers: [{name: severity, value: critical, matchType: =}]
+
+  inhibitRules:
+    - equal: ["alertname", "namespace"]
+      sourceMatch:
+        - name: severity
+          value: critical
+          matchType: =
+        - name: severity
+          value: warning
+          matchType: =~
+
+  receivers:
+    - name: "null"
+    - name: heartbeat
+      webhookConfigs:
+        - urlSecret:
+            name: &secret alertmanager-secret
+            key: ALERTMANAGER_HEARTBEAT_URL
+    - name: pushover
+      pushoverConfigs:
+        - html: true
+          message: |-
+            {{- range .Alerts }}
+              {{- if ne .Annotations.description "" }}
+                {{ .Annotations.description }}
+              {{- else if ne .Annotations.summary "" }}
+                {{ .Annotations.summary }}
+              {{- else if ne .Annotations.message "" }}
+                {{ .Annotations.message }}
+              {{- else }}
+                Alert description not available
+              {{- end }}
+              {{- if gt (len .Labels.SortedPairs) 0 }}
+                <small>
+                {{- range .Labels.SortedPairs }}
+                  <b>{{ .Name }}:</b> {{ .Value }}
+                {{- end }}
+                </small>
+              {{- end }}
+            {{- end }}
+          priority: |-
+            {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+          sendResolved: true
+          sound: gamelan
+          title: >-
+            [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]
+            {{ .CommonLabels.alertname }}
+          ttl: 3600s
+          token:
+            name: *secret
+            key: ALERTMANAGER_PUSHOVER_APP_TOKEN
+          userKey:
+            name: *secret
+            key: ALERTMANAGER_PUSHOVER_USER_KEY
+          urlTitle: View in Alertmanager

kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml

@@ -4,19 +4,18 @@ kind: ExternalSecret
 metadata:
   name: alertmanager
 spec:
-  refreshInterval: 5m
   secretStoreRef:
     kind: ClusterSecretStore
     name: onepassword-connect
   target:
     name: alertmanager-secret
     creationPolicy: Owner
     template:
-      templateFrom:
-        - configMap:
-            name: alertmanager-config-tpl
-            items:
-              - key: alertmanager.yaml
+      engineVersion: v2
+      data:
+        ALERTMANAGER_HEARTBEAT_URL: "{{ .ALERTMANAGER_HEARTBEAT_URL }}"
+        ALERTMANAGER_PUSHOVER_APP_TOKEN: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}"
+        ALERTMANAGER_PUSHOVER_USER_KEY: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}"
   dataFrom:
     - extract:
         key: alertmanager

kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml

@@ -32,8 +32,10 @@ spec:
         ingressClassName: internal
         hosts: ["am.ktwo.io"]
       alertmanagerSpec:
-        useExistingSecret: true
-        configSecret: alertmanager-secret
+        alertmanagerConfiguration:
+          name: alertmanager
+          global:
+            resolveTimeout: 5m
         externalUrl: https://am.ktwo.io
         storage:
           volumeClaimTemplate:

kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml

@@ -2,11 +2,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - ./alertmanagerconfig.yaml
   - ./externalsecret.yaml
   - ./helmrelease.yaml
-configMapGenerator:
-  - name: alertmanager-config-tpl
-    files:
-      - ./resources/alertmanager.yaml
-generatorOptions:
-  disableNameSuffixHash: true

kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml

@@ -11,6 +11,7 @@ spec:
       app.kubernetes.io/name: *app
   dependsOn:
     - name: external-secrets-stores
+    - name: prometheus-operator-crds
     - name: rook-ceph-cluster
   path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app
   prune: true
@@ -42,4 +43,4 @@ spec:
   wait: true
   interval: 30m
   retryInterval: 1m
-  timeout: 15m
+  timeout: 5m

kubernetes/apps/monitoring/kustomization.yaml

@@ -9,6 +9,7 @@ resources:
   - ./kromgo/ks.yaml
   - ./kube-prometheus-stack/ks.yaml
   - ./loki/ks.yaml
+  - ./prometheus-operator-crds/ks.yaml
   - ./promtail/ks.yaml
   - ./unpoller/ks.yaml
   - ./exporters

kubernetes/apps/monitoring/loki/app/helmrelease.yaml

@@ -5,7 +5,6 @@ metadata:
   name: loki
 spec:
   interval: 30m
-  timeout: 15m
   chart:
     spec:
       chart: loki

kubernetes/apps/monitoring/loki/ks.yaml

@@ -19,4 +19,4 @@ spec:
   wait: true
   interval: 30m
   retryInterval: 1m
-  timeout: 15m
+  timeout: 5m

kubernetes/apps/monitoring/prometheus-operator-crds/app/helmrelease.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: prometheus-operator-crds
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: prometheus-operator-crds
+      version: 16.0.1
+      sourceRef:
+        kind: HelmRepository
+        name: prometheus-community
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3

kubernetes/apps/monitoring/prometheus-operator-crds/app/kustomization.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml

kubernetes/apps/monitoring/prometheus-operator-crds/ks.yaml

@@ -0,0 +1,22 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app prometheus-operator-crds
+  namespace: flux-system
+spec:
+  targetNamespace: monitoring
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: rook-ceph-cluster
+  path: ./kubernetes/apps/monitoring/prometheus-operator-crds/app
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: k8s-gitops
+  wait: true
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m

kubernetes/apps/networking/nginx/ks.yaml

@@ -41,7 +41,7 @@ spec:
   wait: true
   interval: 30m
   retryInterval: 1m
-  timeout: 5m
+  timeout: 15m
 ---
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
@@ -63,4 +63,4 @@ spec:
   wait: true
   interval: 30m
   retryInterval: 1m
-  timeout: 5m
+  timeout: 15m