From 4548a2f6ef2718152190d04c18cdbc0d4fb5133d Mon Sep 17 00:00:00 2001
From: Jory Irving <jory@jory.dev>
Date: Wed, 7 Feb 2024 15:00:01 -0700
Subject: [PATCH] migrate alertmanager back to chart

---
 .../karma/app/config/config.yaml              |  2 +-
 .../observability/karma/app/helmrelease.yaml  |  2 +-
 .../app/externalsecret.yaml                   | 31 +++++++++++
 .../app/helmrelease.yaml                      | 52 +++++++++++--------
 .../kube-prometheus-stack/ks.yaml             |  1 +
 .../apps/observability/kustomization.yaml     |  2 +-
 .../teyvat/templates/volsync/claim.yaml       |  2 +
 7 files changed, 67 insertions(+), 25 deletions(-)

diff --git a/kubernetes/teyvat/apps/observability/karma/app/config/config.yaml b/kubernetes/teyvat/apps/observability/karma/app/config/config.yaml
index d51e211ed0..c5fbf92242 100644
--- a/kubernetes/teyvat/apps/observability/karma/app/config/config.yaml
+++ b/kubernetes/teyvat/apps/observability/karma/app/config/config.yaml
@@ -3,7 +3,7 @@ alertmanager:
   interval: 60s
   servers:
     - name: home
-      uri: http://alertmanager.observability.svc.cluster.local:9093
+      uri: http://alertmanager-operated.observability.svc.cluster.local:9093
       timeout: 10s
       healthcheck:
         visible: false
diff --git a/kubernetes/teyvat/apps/observability/karma/app/helmrelease.yaml b/kubernetes/teyvat/apps/observability/karma/app/helmrelease.yaml
index 20866949e5..ec8f11e6c2 100644
--- a/kubernetes/teyvat/apps/observability/karma/app/helmrelease.yaml
+++ b/kubernetes/teyvat/apps/observability/karma/app/helmrelease.yaml
@@ -25,7 +25,7 @@ spec:
   uninstall:
     keepHistory: false
   dependsOn:
-    - name: alertmanager
+    - name: kube-prometheus-stack
       namespace: observability
   values:
     controllers:
diff --git a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/externalsecret.yaml b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/externalsecret.yaml
index 6af01e89e4..655c642edc 100644
--- a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/externalsecret.yaml
+++ b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/externalsecret.yaml
@@ -46,3 +46,34 @@ spec:
       remoteRef:
         key: 5a98804c-6c54-4e09-817e-afd8012c70ad
         property: endpoint
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-secret
+spec:
+  target:
+    deletionPolicy: Delete
+    template:
+      templateFrom:
+        - configMap:
+            name: alertmanager-configmap
+            items:
+              - key: alertmanager.yml
+  data:
+    - secretKey: DISCORD_URL
+      sourceRef:
+        storeRef:
+          name: bitwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 77e30fff-fad7-4812-a635-b034002b5b17
+        property: discord_webhook
+    - secretKey: ALERTMANAGER_HEARTBEAT_URL
+      sourceRef:
+        storeRef:
+          name: bitwarden-fields
+          kind: ClusterSecretStore
+      remoteRef:
+        key: 6893b595-8cc3-47a2-84fa-b0f4016f41bc
+        property: alertmanager_heartbeat
diff --git a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/helmrelease.yaml
index 406761c0b8..47b7611158 100644
--- a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/helmrelease.yaml
+++ b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/app/helmrelease.yaml
@@ -3,7 +3,7 @@
 apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease
 metadata:
-  name: kube-prometheus-stack
+  name: &app kube-prometheus-stack
 spec:
   interval: 30m
   timeout: 15m
@@ -35,9 +35,34 @@ spec:
       valuesKey: kube-state-metrics.yaml
   values:
     crds:
-      enabled: true
     alertmanager:
-      enabled: false
+      ingress:
+        enabled: true
+        pathType: Prefix
+        ingressClassName: internal
+        hosts:
+          - &host alertmanager.${SECRET_DOMAIN}
+        tls:
+          - hosts:
+              - *host
+      alertmanagerSpec:
+        replicas: 2
+        useExistingSecret: true
+        configSecret: alertmanager-secret
+        storage:
+          volumeClaimTemplate:
+            spec:
+              storageClassName: ceph-block
+              resources:
+                requests:
+                  storage: 1Gi
+      affinity: &affinity
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/control-plane
+                    operator: Exists
     grafana:
       enabled: false
       forceDeployDashboards: true
@@ -123,13 +148,7 @@ spec:
                 - __meta_kubernetes_pod_node_name
               targetLabel: kubernetes_node
     prometheusOperator:
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                  - key: node-role.kubernetes.io/control-plane
-                    operator: Exists
+      affinity: *affinity
     prometheus:
       thanosService:
         enabled: true
@@ -157,11 +176,6 @@ spec:
           - auto-gomaxprocs
           - memory-snapshot-on-shutdown
           - new-service-discovery-manager
-        alertingEndpoints:
-          - namespace: observability
-            name: alertmanager
-            port: 9093
-            apiVersion: v2
         externalLabels:
           cluster: ${CLUSTER_NAME}
         replicaExternalLabelName: __replica__
@@ -173,13 +187,7 @@ spec:
         retention: 2d
         retentionSize: 15GB
         replicas: 2
-        affinity:
-          nodeAffinity:
-            requiredDuringSchedulingIgnoredDuringExecution:
-              nodeSelectorTerms:
-                - matchExpressions:
-                    - key: node-role.kubernetes.io/control-plane
-                      operator: Exists
+        affinity: *affinity
         storageSpec:
           volumeClaimTemplate:
             spec:
diff --git a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/ks.yaml b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/ks.yaml
index acb235777b..8b902c63e4 100644
--- a/kubernetes/teyvat/apps/observability/kube-prometheus-stack/ks.yaml
+++ b/kubernetes/teyvat/apps/observability/kube-prometheus-stack/ks.yaml
@@ -21,3 +21,4 @@ spec:
   interval: 30m
   retryInterval: 1m
   timeout: 5m
+  
diff --git a/kubernetes/teyvat/apps/observability/kustomization.yaml b/kubernetes/teyvat/apps/observability/kustomization.yaml
index 8608d11d85..0dcaadc1e9 100644
--- a/kubernetes/teyvat/apps/observability/kustomization.yaml
+++ b/kubernetes/teyvat/apps/observability/kustomization.yaml
@@ -6,7 +6,7 @@ resources:
   # Pre Flux-Kustomizations
   - ./namespace.yaml
   # Flux-Kustomizations
-  - ./alertmanager/ks.yaml
+  #- ./alertmanager/ks.yaml
   - ./gatus/ks.yaml
   - ./grafana/ks.yaml
   - ./karma/ks.yaml
diff --git a/kubernetes/teyvat/templates/volsync/claim.yaml b/kubernetes/teyvat/templates/volsync/claim.yaml
index 423aa08ddc..ddb7926e7c 100644
--- a/kubernetes/teyvat/templates/volsync/claim.yaml
+++ b/kubernetes/teyvat/templates/volsync/claim.yaml
@@ -3,6 +3,8 @@ apiVersion: v1
 kind: PersistentVolumeClaim
 metadata:
   name: "${CLAIM:-${APP}}"
+  labels:
+    app.kubernetes.io/name: "${APP}"
 spec:
   accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
   dataSourceRef: