feat: decentralize loki rules

[onedr0p]

Uses k8s-sidecar feature of loki to move rules into app folders

[bot-ross]

--- kubernetes/main/apps/default/autobrr/app Kustomization: flux-system/autobrr ConfigMap: default/autobrr-loki-rules

+++ kubernetes/main/apps/default/autobrr/app Kustomization: flux-system/autobrr ConfigMap: default/autobrr-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  autobrr.yaml: |
+    ---
+    groups:
+      - name: autobrr
+        rules:
+          - alert: AutobrrNetworkUnhealthy
+            expr: |
+              sum by (app) (count_over_time({app="autobrr"} |~ "(?i)network unhealthy"[2m])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.app }}"
+              summary: "{{ $labels.app }} has a unhealthy network"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: autobrr
+    kustomize.toolkit.fluxcd.io/name: autobrr
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: autobrr-loki-rules
+  namespace: default
+
--- kubernetes/main/apps/default/cross-seed/app Kustomization: flux-system/cross-seed ConfigMap: default/cross-seed-loki-rules

+++ kubernetes/main/apps/default/cross-seed/app Kustomization: flux-system/cross-seed ConfigMap: default/cross-seed-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  cross-seed.yaml: |
+    ---
+    groups:
+      - name: cross-seed
+        rules:
+          - alert: CrossSeedDatabaseMalformed
+            expr: |
+              sum by (app) (count_over_time({app="cross-seed"} |~ "(?i)database disk image is malformed"[1h])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.container }}"
+              summary: "{{ $labels.container }} is experiencing database issues"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: cross-seed
+    kustomize.toolkit.fluxcd.io/name: cross-seed
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: cross-seed-loki-rules
+  namespace: default
+
--- kubernetes/main/apps/default/frigate/app Kustomization: flux-system/frigate ConfigMap: default/frigate-loki-rules

+++ kubernetes/main/apps/default/frigate/app Kustomization: flux-system/frigate ConfigMap: default/frigate-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  frigate.yaml: |
+    ---
+    groups:
+      - name: frigate
+        rules:
+          - alert: FrigateMQTTUnreachable
+            expr: |
+              sum(count_over_time({app="frigate"} |~ "(?i)unable to connect to mqtt server"[2m])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.app }}"
+              summary: "{{ $labels.app }} is unable to reach MQTT"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: frigate
+    kustomize.toolkit.fluxcd.io/name: frigate
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: frigate-loki-rules
+  namespace: default
+
--- kubernetes/main/apps/observability/loki/app Kustomization: flux-system/loki ConfigMap: observability/loki-alerting-rules

+++ kubernetes/main/apps/observability/loki/app Kustomization: flux-system/loki ConfigMap: observability/loki-alerting-rules

@@ -1,208 +0,0 @@

----
-apiVersion: v1
-data:
-  loki-alerting-rules.yaml: |
-    ---
-    groups:
-      - name: zigbee2mqtt
-        rules:
-          - alert: ZigbeeMQTTUnreachable
-            expr: |
-              sum(count_over_time({app="zigbee2mqtt"} |~ "(?i)not connected to mqtt server"[2m])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.app }}"
-              summary: "{{ $labels.app }} is unable to reach MQTT"
-
-      - name: frigate
-        rules:
-          - alert: FrigateMQTTUnreachable
-            expr: |
-              sum(count_over_time({app="frigate"} |~ "(?i)unable to connect to mqtt server"[2m])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.app }}"
-              summary: "{{ $labels.app }} is unable to reach MQTT"
-
-      - name: autobrr
-        rules:
-          - alert: AutobrrNetworkUnhealthy
-            expr: |
-              sum by (app) (count_over_time({app="autobrr"} |~ "(?i)network unhealthy"[2m])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.app }}"
-              summary: "{{ $labels.app }} has a unhealthy network"
-
-      - name: plex
-        rules:
-          - alert: PlexDatabaseBusy
-            expr: |
-              sum by (app) (count_over_time({app="plex"} |~ "(?i)retry busy DB"[2m])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.app }}"
-              summary: "{{ $labels.app }} is experiencing database issues"
-
-      - name: home-assistant
-        rules:
-          - alert: HomeAssistantPostgresUnreachable
-            expr: |
-              sum by (app) (count_over_time({app="home-assistant"} |~ "(?i)error in database connectivity"[2m])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.app }}"
-              summary: "{{ $labels.app }} is unable to connect to postgres"
-
-      - name: cross-seed
-        rules:
-          - alert: CrossSeedDatabaseMalformed
-            expr: |
-              sum by (app) (count_over_time({app="cross-seed"} |~ "(?i)database disk image is malformed"[1h])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.container }}"
-              summary: "{{ $labels.container }} is experiencing database issues"
-
-      - name: qbittorrent
-        rules:
-          - alert: QbittorrentFastResumeRejected
-            expr: |
-              sum by (app) (count_over_time({app="qbittorrent"} |~ "(?i)fast resume rejected"[1h])) > 0
-            for: 2m
-            labels:
-              severity: critical
-              category: logs
-            annotations:
-              app: "{{ $labels.container }}"
-              summary: "{{ $labels.container }} has a torrent with fast resume rejected"
-
-      #
-      # Deprecated
-      #
-
-      # - name: zwave
-      #   rules:
-      #     - alert: ZwaveMQTTUnreachable
-      #       expr: |
-      #         sum(count_over_time({app="zwave"} |~ "(?i)error while connecting mqtt"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.app }} is unable to reach MQTT"
-
-      # - name: node-red
-      #   rules:
-      #     - alert: NodeRedUnableToReachHomeAssistant
-      #       expr: |
-      #         sum by (app) (count_over_time({app="node-red"} |~ "(?i)home assistant.*connecting to undefined"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.app }} is unable to connect to home assistant"
-      #
-      # - name: arr
-      #   rules:
-      #     - alert: ArrDatabaseIsLocked
-      #       expr: |
-      #         sum by (app) (count_over_time({app=~".*arr"} |~ "(?i)database is locked"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.app }} is experiencing locked database issues"
-      #     - alert: ArrDatabaseIsMalformed
-      #       expr: |
-      #         sum by (app) (count_over_time({app=~".*arr"} |~ "(?i)database disk image is malformed"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.app }} is experiencing malformed database disk image issues"
-      #     - alert: ArrFailedToAddTorrent
-      #       expr: |
-      #         avg by (app, message) (count_over_time({app=~".*arr"} |~ "(?i)Couldn't add report to download queue" | json [30m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: warning
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.message }}"
-
-      # - name: bazarr
-      #   rules:
-      #     - alert: BazarrJobRaisedException
-      #       expr: |
-      #         sum by (app) (count_over_time({app="bazarr"} |~ "(?i)Job(.+)Update(.+)from(.+)raised an exception"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: warning
-      #         category: logs
-      #       annotations:
-      #         app: "{{ $labels.app }}"
-      #         summary: "{{ $labels.app }} is raising job exceptions"
-
-      # - name: valetudo
-      #   rules:
-      #     - alert: ValetudoMQTTUnreachable
-      #       expr: |
-      #         sum by (hostname) (count_over_time({hostname="valetudo"} |~ "(?i).*error.*mqtt.*"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         hostname: "{{ $labels.hostname }}"
-      #         summary: "{{ $labels.hostname }} is unable to connect to MQTT"
-
-      # - name: smart
-      #   rules:
-      #     - alert: SMARTFailure
-      #       expr: |
-      #         sum by (hostname) (count_over_time({hostname=~".+"} | json | _SYSTEMD_UNIT = "smartmontools.service" !~ "(?i)previous self-test completed without error" !~ "(?i)Prefailure" |~ "(?i)(error|fail)"[2m])) > 0
-      #       for: 2m
-      #       labels:
-      #         severity: critical
-      #         category: logs
-      #       annotations:
-      #         hostname: "{{ $labels.hostname }}"
-      #         summary: "{{ $labels.hostname }} has reported SMART failures"
-kind: ConfigMap
-metadata:
-  labels:
-    app.kubernetes.io/name: loki
-    kustomize.toolkit.fluxcd.io/name: loki
-    kustomize.toolkit.fluxcd.io/namespace: flux-system
-    loki_rule: 'true'
-  name: loki-alerting-rules
-  namespace: observability
-
--- kubernetes/main/apps/default/plex/app Kustomization: flux-system/plex ConfigMap: default/plex-loki-rules

+++ kubernetes/main/apps/default/plex/app Kustomization: flux-system/plex ConfigMap: default/plex-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  plex.yaml: |
+    ---
+    groups:
+      - name: plex
+        rules:
+          - alert: PlexDatabaseBusy
+            expr: |
+              sum by (app) (count_over_time({app="plex"} |~ "(?i)retry busy DB"[2m])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.app }}"
+              summary: "{{ $labels.app }} is experiencing database issues"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: plex
+    kustomize.toolkit.fluxcd.io/name: plex
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: plex-loki-rules
+  namespace: default
+
--- kubernetes/main/apps/default/qbittorrent/app Kustomization: flux-system/qbittorrent ConfigMap: default/qbittorrent-loki-rules

+++ kubernetes/main/apps/default/qbittorrent/app Kustomization: flux-system/qbittorrent ConfigMap: default/qbittorrent-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  qbittorrent.yaml: |
+    ---
+    groups:
+      - name: qbittorrent
+        rules:
+          - alert: QbittorrentFastResumeRejected
+            expr: |
+              sum by (app) (count_over_time({app="qbittorrent"} |~ "(?i)fast resume rejected"[1h])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.container }}"
+              summary: "{{ $labels.container }} has a torrent with fast resume rejected"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: qbittorrent
+    kustomize.toolkit.fluxcd.io/name: qbittorrent
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: qbittorrent-loki-rules
+  namespace: default
+
--- kubernetes/main/apps/default/zigbee2mqtt/app Kustomization: flux-system/zigbee2mqtt ConfigMap: default/zigbee2mqtt-loki-rules

+++ kubernetes/main/apps/default/zigbee2mqtt/app Kustomization: flux-system/zigbee2mqtt ConfigMap: default/zigbee2mqtt-loki-rules

@@ -0,0 +1,28 @@

+---
+apiVersion: v1
+data:
+  zigbee2mqtt.yaml: |
+    ---
+    groups:
+      - name: zigbee2mqtt
+        rules:
+          - alert: ZigbeeMQTTUnreachable
+            expr: |
+              sum(count_over_time({app="zigbee2mqtt"} |~ "(?i)not connected to mqtt server"[2m])) > 0
+            for: 2m
+            labels:
+              severity: critical
+              category: logs
+            annotations:
+              app: "{{ $labels.app }}"
+              summary: "{{ $labels.app }} is unable to reach MQTT"
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: zigbee2mqtt
+    kustomize.toolkit.fluxcd.io/name: zigbee2mqtt
+    kustomize.toolkit.fluxcd.io/namespace: flux-system
+    loki_rule: 'true'
+  name: zigbee2mqtt-loki-rules
+  namespace: default
+