fix(prom): update prom-stack config

willianpaixao · Nov 21, 2024 · a7ecf0f · a7ecf0f
1 parent d8ac121
commit a7ecf0f
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 2 deletions.
diff --git a/kubernetes/raspberry/flux-system/cluster-secrets.sops.yaml b/kubernetes/raspberry/flux-system/cluster-secrets.sops.yaml
@@ -4,6 +4,7 @@ metadata:
     name: cluster-secrets
     namespace: flux-system
 stringData:
+    TELEGRAM_CHAT_ID: ENC[AES256_GCM,data:UYI7A4cnoMW6LB2u2rk=,iv:GRKlSB4W1GHTRXkA9fyTWqJZcjbFJVPgaWIZjxPlgvM=,tag:m1Gb/OGgFCI3KYm2YJ7jUQ==,type:str]
     SECRET_DOMAIN: ENC[AES256_GCM,data:yxxmAaiQIgNNKuTP,iv:ijCjY0DZPDt2u/gjMxQ91V+a3okd/7J5rmSNqaABawM=,tag:1cEBaYTiQMxF6sJd1dSpWw==,type:str]
     SECRET_ACME_EMAIL: ENC[AES256_GCM,data:NROOEvv2p/BqnKG5OVYv,iv:tLsrO8xW4rTEdaHm9bE60w1+pVtlQkmh3nfkOa6TK4Y=,tag:cY3VDLexaCOqwBZ+7hbM6g==,type:str]
     SECRET_CLOUDFLARE_TUNNEL_ID: ENC[AES256_GCM,data:n91Brd4fEXGq0JhoHOBOOj6EAoRkVqk4w1VoqM/JUHOO8hgN,iv:ssaWwZdr9efj5TBQ6XB8TVbpSABlXqDQMC0sp8k1eAk=,tag:hJMlfE/ZJ6opKUbx6I6SIQ==,type:str]
@@ -31,8 +32,8 @@ sops:
             Tys4alpFdlVHZVBqeEIxaHgyTXJrSlEKayFLG7MILTO8Dapjn90S0pvFYDmSvlPQ
             hVPLuWEwNNNVSfuRLPkQi3spegTCeyos3k+e0OQmdU1h1+W8CON/6A==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-24T11:18:41Z"
-    mac: ENC[AES256_GCM,data:qh3OXT9opRkUqhy3xn2AVFSFz/B62VBOfppkr5mYxanOgFjz5RIqQlbBoSZTMq/0Dg1tc0oxxfv1seScatbROObdRSvDFbrDq82z8mKHVFxGm70bwEp1ZsZy2JUEyA6df3XPUuEivzcSvudivK1diBz89P0EpSvPpeb73LjHiaI=,iv:6wtwADYNqgZ98wb3Mu/PMeUQdFMmtUQIU7sZB07anPM=,tag:qaB403+Uk5IAsBigO4IJHA==,type:str]
+    lastmodified: "2024-11-21T17:13:06Z"
+    mac: ENC[AES256_GCM,data:Az/we+sAPm8TCKODAwD5QLcCit8Hfw0cnkWN9jl3m6bS/+CcFhL2/4dtMtuxcUCQ3cXiBHcL+HYZlVsrxV1Mw/7vM+kNBaTY/yTnGfb5WUG7dkRpwXGjUdL86t5jGJV72fsmVMTPB1mXP61D9EKbbqykj4Vpp8/L4bOoyr85wVg=,iv:N9Ju8uSzt6WCBaCstH6DcL3s91cnSS+3ghHNTxUFhPA=,tag:mlgrbp4tmbI3yRY5S1wYtg==,type:str]
     pgp: []
     encrypted_regex: ^(data|stringData)$
     version: 3.9.1
diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/alertmanager-config.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/alertmanager-config.yaml
@@ -0,0 +1,22 @@
+---
+apiVersion: monitoring.coreos.com/v1alpha1
+kind: AlertmanagerConfig
+metadata:
+  name: alertmanager-config
+  namespace: observability
+  labels:
+    alertmanagerConfig: kube-prometheus-stack
+spec:
+  route:
+    groupBy: ['alertname']
+    groupWait: 30s
+    groupInterval: 5m
+    repeatInterval: 12h
+    receiver: 'telegram'
+  receivers:
+    - name: telegram
+      telegramConfigs:
+        botToken:
+          name: 'telegram-config'
+          key: 'BOT_TOKEN'
+        chatID: -1002084680078
diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/helmrelease.yaml
@@ -45,6 +45,8 @@ spec:
               - *host
       alertmanagerSpec:
         replicas: 1
+        alertmanagerConfiguration:
+          name: alertmanager-config
     prometheus:
       service:
         type: LoadBalancer
@@ -67,6 +69,10 @@ spec:
         tls:
           - hosts:
               - *host
+      serviceMonitor:
+        relabelings:
+          - replacement: raspberry
+            targetLabel: cluster
       prometheusSpec:
         replicas: 1
         scrapeInterval: 30s
@@ -94,6 +100,60 @@ spec:
               resources:
                 requests:
                   storage: 64Gi
+    kubeApiServer:
+      serviceMonitor:
+        relabelings:
+          - replacement: raspberry
+            targetLabel: cluster
+    kubelet:
+      serviceMonitor:
+        cAdvisorRelabelings:
+          - sourceLabels: [__metrics_path__]
+            targetLabel: metrics_path
+          - replacement: raspberry
+            targetLabel: cluster
+        relabelings:
+          - sourceLabels: [__metrics_path__]
+            targetLabel: metrics_path
+          - replacement: raspberry
+            targetLabel: cluster
+    kubeControllerManager:
+      endpoints:
+        - 192.168.0.5
+        - 192.168.0.6
+        - 192.168.0.7
+      serviceMonitor:
+        relabelings:
+          - replacement: raspberry
+            targetLabel: cluster
+    kubeScheduler:
+      endpoints:
+        - 192.168.0.5
+        - 192.168.0.6
+        - 192.168.0.7
+      serviceMonitor:
+        relabelings:
+          - replacement: raspberry
+            targetLabel: cluster
+    kubeProxy:
+      enabled: false
+    kube-state-metrics:
+      prometheus:
+        monitor:
+          relabelings:
+            - replacement: raspberry
+              targetLabel: cluster
+    prometheus-node-exporter:
+      prometheus:
+        monitor:
+          relabelings:
+            - replacement: raspberry
+              targetLabel: cluster
+    prometheusOperator:
+      serviceMonitor:
+        relabelings:
+          - replacement: raspberry
+            targetLabel: cluster
     grafana:
       enabled: true
       defaultDashboardsTimezone: ${TIMEZONE}
@@ -114,6 +174,8 @@ spec:
           multicluster:
             global:
               enabled: true
+      plugins:
+        - grafana-strava-datasource
       persistence:
         enabled: true
         storageClassName: longhorn

diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/kustomization.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/kustomization.yaml
@@ -2,4 +2,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - alertmanager-config.yaml
   - helmrelease.yaml
+  - telegram-config.sops.yaml
diff --git a/kubernetes/raspberry/observability/kube-prometheus-stack/app/telegram-config.sops.yaml b/kubernetes/raspberry/observability/kube-prometheus-stack/app/telegram-config.sops.yaml
@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: telegram-config
+    namespace: observability
+data:
+    BOT_TOKEN: ENC[AES256_GCM,data:p6+Ju1AIuBT+O4uJ7ZQS7+KftQ18qqrhV7x5p5T6Jci+xUUMrbcWJUukCO4=,iv:Xh59vtEMq/EGiWSauTLd2/24Fwvy1Nac1MmJjxlh7WU=,tag:dNKWx+rh8Eo2AVOb+BnUUQ==,type:str]
+    CHAT_ID: ENC[AES256_GCM,data:Iy8A7A4PvyShJM1TXpNONMzc+ro=,iv:qLdTA/uS6rr7eoCVARl/c/nzBABvGT1x6l8eyQdU260=,tag:DK91+mMXIXW5Za2F0D+AKQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1nkvss2a8xvmjauvr5mxzm233hyh2mk2fg4s6pt0t0kcn03dv34wqtgymg8
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkZzB1bjh1alJkKyt1aWNl
+            eWdMYjZqbjMwMWNiYnB4R0ZXZE82c3pFZzJZClhxUEh4UTFlcFZnVTJBVmZ2eFMz
+            blBWRnBmeEdnNjBISUJYQTBtalZWR3MKLS0tIERmemFSQ0o0bk1MT0dIWU9iOXN1
+            R2lsSUFvMjFQVTRYT25tb3FOMG9NQTgKAl8sBHxxpvLC3zVTMoEUe/dLPzyRQjdu
+            iaYlewpypwzduLlDfo2dpfErOCedf6oC2w4gRmNJTP0lg2KWdzN33A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1wxwqdrmkwkzsxajp58g0cgeextgf4wq287fv82pptv9yghkfgcqql66zhj
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxTWEwb29QbW0yUjk2QUlN
+            djVrbnZPeUgvMmJwZU5MZ0ozUTBnN0RQcEdjClRaeEZSVU5SZHB2aHVoczBjOXNs
+            VVlyZU5aVTlxcldaTXBFYVRyaFpDSmMKLS0tIGFidm9wVW1YT2NMdy9rSnZENytj
+            UVc3WTRVSFFVSGNrNCt0MXFYUWZLbDQKpIJimfVIW3uR4TBL/emtALvJmKdYkh4t
+            M1UOLPI4cDPK8vC6mgg6fZlJyzp0A3Q8kPoTV2STjjh9Ezpk2blVkQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-11-21T17:00:48Z"
+    mac: ENC[AES256_GCM,data:c6jx3bhaqqxt/lPzq8P5QSZx1D7dlbfu6yqjhzdsNNyiL9muymDtCLpfDE/Nfppq/ripLK99EOI52/j+8jlLWOFeARm89b+08DUFUR/NT2bEz73S+dMeNeuFUItSyTH58n9NEd/pnkksY1+DlbgCELNQnwnOyP7uQQBvD1LIEKQ=,iv:icX6P1SHjZHe4VEMaybqbROeFONhkl1oI3DmrCTQ0+M=,tag:8YYAI0JY+qQNShRlwHzYRQ==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.9.1
diff --git a/kubernetes/turing/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/turing/observability/kube-prometheus-stack/app/helmrelease.yaml
@@ -27,6 +27,24 @@ spec:
       strategy: rollback
       retries: 3
   values:
+    alertmanager:
+      enabled: false
+    grafana:
+      enabled: false
+    kubeEtcd:
+      enabled: false
+    kubeControllerManager:
+      endpoints:
+        - 192.168.0.171
+        - 192.168.0.172
+        - 192.168.0.173
+    kubeScheduler:
+      endpoints:
+        - 192.168.0.171
+        - 192.168.0.172
+        - 192.168.0.173
+    kubeProxy:
+      enabled: false
     prometheus:
       agentMode: true
       prometheusSpec: