diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml index 7550251b65..c21a66c0e4 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml @@ -32,8 +32,10 @@ spec: ingressClassName: internal hosts: ["am.ktwo.io"] alertmanagerSpec: - useExistingSecret: true - configSecret: alertmanager-secret + alertmanagerConfiguration: + name: alertmanager + global: + resolveTimeout: 5m externalUrl: https://am.ktwo.io storage: volumeClaimTemplate: diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml index 004b793183..5dd7baca73 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/app/kustomization.yaml @@ -2,11 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ./externalsecret.yaml - ./helmrelease.yaml -configMapGenerator: - - name: alertmanager-config-tpl - files: - - ./resources/alertmanager.yaml -generatorOptions: - disableNameSuffixHash: true diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml deleted file mode 100644 index fd5990ce5a..0000000000 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/resources/alertmanager.yaml +++ /dev/null @@ -1,71 +0,0 @@ ---- -global: - resolve_timeout: 5m - -route: - group_by: ["alertname", "job"] - group_interval: 10m - group_wait: 1m - receiver: pushover - repeat_interval: 12h - routes: - - receiver: heartbeat - group_interval: 15s - group_wait: 0s - repeat_interval: 5m - matchers: - - alertname =~ "Watchdog" - - receiver: "null" - matchers: - - alertname =~ "InfoInhibitor" - - receiver: pushover - continue: true - matchers: - - severity = "critical" - -inhibit_rules: - - equal: ["alertname", "namespace"] - source_matchers: - - severity = "critical" - target_matchers: - - severity = "warning" - -receivers: - - name: heartbeat - webhook_configs: - - send_resolved: true - url: "{{ .ALERTMANAGER_HEARTBEAT_URL }}" - - name: "null" - - name: pushover - pushover_configs: - - html: true - message: |- - {{ "{{-" }} range .Alerts {{ "}}" }} - {{ "{{-" }} if ne .Annotations.description "" {{ "}}" }} - {{ "{{" }} .Annotations.description {{ "}}" }} - {{ "{{-" }} else if ne .Annotations.summary "" {{ "}}" }} - {{ "{{" }} .Annotations.summary {{ "}}" }} - {{ "{{-" }} else if ne .Annotations.message "" {{ "}}" }} - {{ "{{" }} .Annotations.message {{ "}}" }} - {{ "{{-" }} else {{ "}}" }} - Alert description not available - {{ "{{-" }} end {{ "}}" }} - {{ "{{-" }} if gt (len .Labels.SortedPairs) 0 {{ "}}" }} - - {{ "{{-" }} range .Labels.SortedPairs {{ "}}" }} - {{ "{{" }} .Name {{ "}}" }}: {{ "{{" }} .Value {{ "}}" }} - {{ "{{-" }} end {{ "}}" }} - - {{ "{{-" }} end {{ "}}" }} - {{ "{{-" }} end {{ "}}" }} - priority: |- - {{ "{{" }} if eq .Status "firing" {{ "}}" }}1{{ "{{" }} else {{ "}}" }}0{{ "{{" }} end {{ "}}" }} - send_resolved: true - sound: gamelan - title: >- - [{{ "{{" }} .Status | toUpper {{ "}}" }}{{ "{{" }} if eq .Status "firing" {{ "}}" }}:{{ "{{" }} .Alerts.Firing | len {{ "}}" }}{{ "{{" }} end {{ "}}" }}] - {{ "{{" }} .CommonLabels.alertname {{ "}}" }} - token: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}" - # ttl: "{{ .ALERTMANAGER_PUSHOVER_TTL }}" - url_title: View in Alertmanager - user_key: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}" diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml new file mode 100644 index 0000000000..b06279be8f --- /dev/null +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/alertmanagerconfig.yaml @@ -0,0 +1,77 @@ +--- +apiVersion: monitoring.coreos.com/v1alpha1 +kind: AlertmanagerConfig +metadata: + name: alertmanager +spec: + route: + groupBy: ["alertname", "job"] + groupInterval: 10m + groupWait: 1m + receiver: pushover + repeatInterval: 12h + routes: + - receiver: "null" + matchers: [{name: alertname, value: InfoInhibitor, matchType: =}] + - receiver: heartbeat + groupInterval: 15s + groupWait: 0s + repeatInterval: 5m + matchers: [{name: alertname, value: Watchdog, matchType: =}] + - receiver: pushover + matchers: [{name: severity, value: critical, matchType: =}] + + inhibitRules: + - equal: ["alertname", "namespace"] + sourceMatch: + - name: severity + value: critical + matchType: = + - name: severity + value: warning + matchType: =~ + + receivers: + - name: "null" + - name: heartbeat + webhookConfigs: + - urlSecret: + name: &secret alertmanager-secret + key: ALERTMANAGER_HEARTBEAT_URL + - name: pushover + pushoverConfigs: + - html: true + message: |- + {{- range .Alerts }} + {{- if ne .Annotations.description "" }} + {{ .Annotations.description }} + {{- else if ne .Annotations.summary "" }} + {{ .Annotations.summary }} + {{- else if ne .Annotations.message "" }} + {{ .Annotations.message }} + {{- else }} + Alert description not available + {{- end }} + {{- if gt (len .Labels.SortedPairs) 0 }} + + {{- range .Labels.SortedPairs }} + {{ .Name }}: {{ .Value }} + {{- end }} + + {{- end }} + {{- end }} + priority: |- + {{ if eq .Status "firing" }}1{{ else }}0{{ end }} + sendResolved: true + sound: gamelan + title: >- + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] + {{ .CommonLabels.alertname }} + ttl: 3600s + token: + name: *secret + key: ALERTMANAGER_PUSHOVER_APP_TOKEN + userKey: + name: *secret + key: ALERTMANAGER_PUSHOVER_USER_KEY + urlTitle: View in Alertmanager diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml similarity index 53% rename from kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml rename to kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml index 5b44b3c860..c431723ba0 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/externalsecret.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/externalsecret.yaml @@ -4,7 +4,6 @@ kind: ExternalSecret metadata: name: alertmanager spec: - refreshInterval: 5m secretStoreRef: kind: ClusterSecretStore name: onepassword-connect @@ -12,11 +11,11 @@ spec: name: alertmanager-secret creationPolicy: Owner template: - templateFrom: - - configMap: - name: alertmanager-config-tpl - items: - - key: alertmanager.yaml + engineVersion: v2 + data: + ALERTMANAGER_HEARTBEAT_URL: "{{ .ALERTMANAGER_HEARTBEAT_URL }}" + ALERTMANAGER_PUSHOVER_APP_TOKEN: "{{ .ALERTMANAGER_PUSHOVER_APP_TOKEN }}" + ALERTMANAGER_PUSHOVER_USER_KEY: "{{ .ALERTMANAGER_PUSHOVER_USER_KEY }}" dataFrom: - extract: key: alertmanager diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml new file mode 100644 index 0000000000..853ffc1d4f --- /dev/null +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/configs/kustomization.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./alertmanagerconfig.yaml + - ./externalsecret.yaml diff --git a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml index bcaae4b300..ecdf9263f4 100644 --- a/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml +++ b/kubernetes/apps/monitoring/kube-prometheus-stack/ks.yaml @@ -10,7 +10,6 @@ spec: labels: app.kubernetes.io/name: *app dependsOn: - - name: external-secrets-stores - name: rook-ceph-cluster path: ./kubernetes/apps/monitoring/kube-prometheus-stack/app prune: true @@ -24,6 +23,29 @@ spec: --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization +metadata: + name: &app kube-prometheus-stack-configs + namespace: flux-system +spec: + targetNamespace: monitoring + commonMetadata: + labels: + app.kubernetes.io/name: *app + dependsOn: + - name: external-secrets-stores + - name: kube-prometheus-stack + path: ./kubernetes/apps/monitoring/kube-prometheus-stack/configs + prune: true + sourceRef: + kind: GitRepository + name: k8s-gitops + wait: true + interval: 30m + retryInterval: 1m + timeout: 15m +--- +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization metadata: name: &app kube-prometheus-stack-tools namespace: flux-system