Add relabeling rules to pod logs (#22)

Signed-off-by: Pete Wall <[email protected]>
grafana · Jul 7, 2023 · 28f9425 · 28f9425
1 parent 0216042
commit 28f9425
Show file tree

Hide file tree

Showing 8 changed files with 156 additions and 40 deletions.
diff --git a/charts/k8s-monitoring/Chart.yaml b/charts/k8s-monitoring/Chart.yaml
@@ -3,8 +3,8 @@ name: k8s-monitoring
 description: A Helm chart for gathering, scraping, and forwarding Kubernetes infrastructure metrics and logs to a Grafana Stack.
 type: application
 
-version: 0.0.6
-appVersion: 1.1.5
+version: 0.0.7
+appVersion: 1.1.6
 icon: https://raw.githubusercontent.com/grafana/grafana/main/public/img/grafana_icon.svg
 maintainers:
   - email: [email protected]

diff --git a/charts/k8s-monitoring/README.md b/charts/k8s-monitoring/README.md
@@ -1,6 +1,6 @@
 # k8s-monitoring
 
-![Version: 0.0.6](https://img.shields.io/badge/Version-0.0.6-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.5](https://img.shields.io/badge/AppVersion-1.1.5-informational?style=flat-square)
+![Version: 0.0.7](https://img.shields.io/badge/Version-0.0.7-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 1.1.6](https://img.shields.io/badge/AppVersion-1.1.6-informational?style=flat-square)
 
 A Helm chart for gathering, scraping, and forwarding Kubernetes infrastructure metrics and logs to a Grafana Stack.
 

diff --git a/charts/k8s-monitoring/templates/agent_config/_cluster_events.river.txt b/charts/k8s-monitoring/templates/agent_config/_cluster_events.river.txt
@@ -2,6 +2,16 @@
 // Cluster Events
 loki.source.kubernetes_events "cluster_events" {
   job_name   = "integrations/kubernetes/eventhandler"
-  forward_to = [loki.process.add_cluster_label.receiver]
+  forward_to = [loki.process.cluster_events.receiver]
+}
+
+loki.process "cluster_events" {
+  stage.static_labels {
+    values = {
+      cluster = local.file.cluster_name.content,
+    }
+  }
+
+  forward_to = [loki.write.grafana_cloud_loki.receiver]
 }
 {{ end }}
diff --git a/charts/k8s-monitoring/templates/agent_config/_loki.river.txt b/charts/k8s-monitoring/templates/agent_config/_loki.river.txt
@@ -1,14 +1,5 @@
 {{ define "agent.config.loki" }}
 // Grafana Cloud Loki
-loki.process "add_cluster_label" {
-  stage.static_labels {
-    values = {
-      cluster = local.file.cluster_name.content,
-    }
-  }
-  forward_to = [loki.write.grafana_cloud_loki.receiver]
-}
-
 local.file "loki_host" {
   filename  = "/etc/grafana-agent-credentials/loki_host"
 }

diff --git a/charts/k8s-monitoring/templates/agent_config/_pod_logs.river.txt b/charts/k8s-monitoring/templates/agent_config/_pod_logs.river.txt
@@ -1,17 +1,45 @@
 {{ define "agent.config.logs.pod_logs" }}
 // Pod Logs
+discovery.relabel "pod_logs" {
+  targets = discovery.kubernetes.pods.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    action = "replace"
+    target_label = "namespace"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    action = "replace"
+    target_label = "pod"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    action = "replace"
+    target_label = "container"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
+    separator = "/"
+    action = "replace"
+    replacement = "$1"
+    target_label = "job"
+  }
+}
+
 loki.source.kubernetes "pod_logs" {
-  targets    = discovery.kubernetes.pods.targets
+  targets    = discovery.relabel.pod_logs.output
   forward_to = [loki.process.pod_logs.receiver]
 }
 
 loki.process "pod_logs" {
   stage.static_labels {
     values = {
-      job = "integration/kubernetes/pod_logs",
+      cluster = local.file.cluster_name.content,
     }
   }
+
   stage.{{- .Values.logs.pod_logs.loggingFormat }} {}
-  forward_to = [loki.process.add_cluster_label.receiver]
+  forward_to = [loki.write.grafana_cloud_loki.receiver]
 }
 {{ end }}
diff --git a/tests/default-values/output.yaml b/tests/default-values/output.yaml
@@ -373,37 +373,66 @@ data:
     }
 
     // Pod Logs
+    discovery.relabel "pod_logs" {
+      targets = discovery.kubernetes.pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        action = "replace"
+        target_label = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        action = "replace"
+        target_label = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        action = "replace"
+        target_label = "container"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
+        separator = "/"
+        action = "replace"
+        replacement = "$1"
+        target_label = "job"
+      }
+    }
+
     loki.source.kubernetes "pod_logs" {
-      targets    = discovery.kubernetes.pods.targets
+      targets    = discovery.relabel.pod_logs.output
       forward_to = [loki.process.pod_logs.receiver]
     }
 
     loki.process "pod_logs" {
       stage.static_labels {
         values = {
-          job = "integration/kubernetes/pod_logs",
+          cluster = local.file.cluster_name.content,
         }
       }
+
       stage.docker {}
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
 
     // Cluster Events
     loki.source.kubernetes_events "cluster_events" {
       job_name   = "integrations/kubernetes/eventhandler"
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.process.cluster_events.receiver]
     }
-
-    // Grafana Cloud Loki
-    loki.process "add_cluster_label" {
+
+    loki.process "cluster_events" {
       stage.static_labels {
         values = {
           cluster = local.file.cluster_name.content,
         }
       }
+
       forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
-
+
+    // Grafana Cloud Loki
     local.file "loki_host" {
       filename  = "/etc/grafana-agent-credentials/loki_host"
     }

diff --git a/tests/logs-only/output.yaml b/tests/logs-only/output.yaml
@@ -49,37 +49,66 @@ data:
     }
         
     // Pod Logs
+    discovery.relabel "pod_logs" {
+      targets = discovery.kubernetes.pods.targets
+    
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        action = "replace"
+        target_label = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        action = "replace"
+        target_label = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        action = "replace"
+        target_label = "container"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
+        separator = "/"
+        action = "replace"
+        replacement = "$1"
+        target_label = "job"
+      }
+    }
+    
     loki.source.kubernetes "pod_logs" {
-      targets    = discovery.kubernetes.pods.targets
+      targets    = discovery.relabel.pod_logs.output
       forward_to = [loki.process.pod_logs.receiver]
     }
     
     loki.process "pod_logs" {
       stage.static_labels {
         values = {
-          job = "integration/kubernetes/pod_logs",
+          cluster = local.file.cluster_name.content,
         }
       }
+    
       stage.docker {}
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
         
     // Cluster Events
     loki.source.kubernetes_events "cluster_events" {
       job_name   = "integrations/kubernetes/eventhandler"
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.process.cluster_events.receiver]
     }
-        
-    // Grafana Cloud Loki
-    loki.process "add_cluster_label" {
+    
+    loki.process "cluster_events" {
       stage.static_labels {
         values = {
           cluster = local.file.cluster_name.content,
         }
       }
+    
       forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
-    
+        
+    // Grafana Cloud Loki
     local.file "loki_host" {
       filename  = "/etc/grafana-agent-credentials/loki_host"
     }

diff --git a/tests/openshift-compatible/output.yaml b/tests/openshift-compatible/output.yaml
@@ -342,37 +342,66 @@ data:
     }
 
     // Pod Logs
+    discovery.relabel "pod_logs" {
+      targets = discovery.kubernetes.pods.targets
+
+      rule {
+        source_labels = ["__meta_kubernetes_namespace"]
+        action = "replace"
+        target_label = "namespace"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_name"]
+        action = "replace"
+        target_label = "pod"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_pod_container_name"]
+        action = "replace"
+        target_label = "container"
+      }
+      rule {
+        source_labels = ["__meta_kubernetes_namespace", "__meta_kubernetes_pod_name"]
+        separator = "/"
+        action = "replace"
+        replacement = "$1"
+        target_label = "job"
+      }
+    }
+
     loki.source.kubernetes "pod_logs" {
-      targets    = discovery.kubernetes.pods.targets
+      targets    = discovery.relabel.pod_logs.output
       forward_to = [loki.process.pod_logs.receiver]
     }
 
     loki.process "pod_logs" {
       stage.static_labels {
         values = {
-          job = "integration/kubernetes/pod_logs",
+          cluster = local.file.cluster_name.content,
         }
       }
+
       stage.docker {}
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
 
     // Cluster Events
     loki.source.kubernetes_events "cluster_events" {
       job_name   = "integrations/kubernetes/eventhandler"
-      forward_to = [loki.process.add_cluster_label.receiver]
+      forward_to = [loki.process.cluster_events.receiver]
     }
-
-    // Grafana Cloud Loki
-    loki.process "add_cluster_label" {
+
+    loki.process "cluster_events" {
       stage.static_labels {
         values = {
           cluster = local.file.cluster_name.content,
         }
       }
+
       forward_to = [loki.write.grafana_cloud_loki.receiver]
     }
-
+
+    // Grafana Cloud Loki
     local.file "loki_host" {
       filename  = "/etc/grafana-agent-credentials/loki_host"
     }