From 144d36bbbe68e38955a6c1aff498cc64d70c69bd Mon Sep 17 00:00:00 2001
From: Nicolas Lamirault <nicolas.lamirault@gmail.com>
Date: Mon, 8 Jul 2024 18:04:49 +0200
Subject: [PATCH] [fix] Helm : Grafana dashboard for k8s metacollector

Signed-off-by: Nicolas Lamirault <nicolas.lamirault@gmail.com>
Signed-off-by: Leonardo Grasso <me@leonardograsso.com>
---
 .../k8s-metacollector-dashboard.json          | 566 ++++++++++--------
 1 file changed, 308 insertions(+), 258 deletions(-)

diff --git a/charts/k8s-metacollector/dashboards/k8s-metacollector-dashboard.json b/charts/k8s-metacollector/dashboards/k8s-metacollector-dashboard.json
index 62b721df4..c9682d203 100644
--- a/charts/k8s-metacollector/dashboards/k8s-metacollector-dashboard.json
+++ b/charts/k8s-metacollector/dashboards/k8s-metacollector-dashboard.json
@@ -25,14 +25,14 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 17,
+  "id": 146,
   "links": [],
   "liveNow": false,
   "panels": [
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -40,6 +40,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -53,6 +54,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 3,
             "pointSize": 5,
@@ -95,7 +97,6 @@
       },
       "id": 2,
       "interval": "1m",
-      "links": [],
       "options": {
         "legend": {
           "calcs": [],
@@ -104,6 +105,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -113,7 +115,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "code",
           "exemplar": true,
@@ -133,7 +135,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -141,6 +143,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -154,6 +157,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 3,
             "pointSize": 5,
@@ -196,7 +200,6 @@
       },
       "id": 4,
       "interval": "1m",
-      "links": [],
       "options": {
         "legend": {
           "calcs": [],
@@ -205,6 +208,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -214,7 +218,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "exemplar": true,
@@ -234,7 +238,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "description": "Number of subscribers",
       "fieldConfig": {
@@ -244,6 +248,7 @@
             "mode": "fixed"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -257,6 +262,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -305,6 +311,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -314,7 +321,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "expr": "meta_collector_server_subscribers{namespace=\"$namespace\", pod=\"$pod\", job=\"$job\"}",
@@ -329,7 +336,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -377,6 +384,7 @@
           "values": false
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -386,7 +394,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "expr": "sum by(type) (meta_collector_broker_queue_adds{pod=\"$pod\", namespace=\"$namespace\"})",
@@ -401,7 +409,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -423,7 +431,7 @@
       "gridPos": {
         "h": 8,
         "w": 12,
-        "x": 7,
+        "x": 0,
         "y": 16
       },
       "id": 23,
@@ -447,6 +455,7 @@
           "values": false
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -456,7 +465,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "expr": "sum by(controller) (controller_runtime_reconcile_total{pod=\"$pod\", namespace=\"$namespace\"})",
@@ -471,72 +480,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
-      },
-      "description": "Events sent to subscribers",
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green",
-                "value": null
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 24
-      },
-      "id": 17,
-      "options": {
-        "displayMode": "gradient",
-        "minVizHeight": 10,
-        "minVizWidth": 0,
-        "orientation": "horizontal",
-        "reduceOptions": {
-          "calcs": [
-            "lastNotNull"
-          ],
-          "fields": "",
-          "values": false
-        },
-        "showUnfilled": true,
-        "valueMode": "color"
-      },
-      "pluginVersion": "9.5.1",
-      "targets": [
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prometheus"
-          },
-          "editorMode": "builder",
-          "exemplar": false,
-          "expr": "sum by(kind) (meta_collector_broker_dispatched_events{pod=\"$pod\", namespace=\"$namespace\"})",
-          "format": "time_series",
-          "legendFormat": "{{kind}}",
-          "range": true,
-          "refId": "A"
-        }
-      ],
-      "title": "Events Dispatched Per Resource Kind",
-      "type": "bargauge"
-    },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -564,7 +508,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 24
+        "y": 16
       },
       "id": 26,
       "options": {
@@ -579,14 +523,16 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.5.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "expr": "sum by(job) (meta_collector_broker_dispatched_events{pod=\"$pod\", namespace=\"$namespace\"})",
@@ -601,13 +547,13 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
+      "description": "Events sent to subscribers",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "fixedColor": "semi-dark-orange",
-            "mode": "fixed"
+            "mode": "thresholds"
           },
           "mappings": [],
           "thresholds": {
@@ -626,13 +572,15 @@
         "h": 8,
         "w": 12,
         "x": 0,
-        "y": 32
+        "y": 24
       },
-      "id": 24,
+      "id": 17,
       "options": {
         "displayMode": "gradient",
+        "maxVizHeight": 300,
         "minVizHeight": 10,
         "minVizWidth": 0,
+        "namePlacement": "auto",
         "orientation": "horizontal",
         "reduceOptions": {
           "calcs": [
@@ -642,29 +590,32 @@
           "values": false
         },
         "showUnfilled": true,
+        "sizing": "auto",
         "valueMode": "color"
       },
-      "pluginVersion": "9.5.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
-          "expr": "sum by(name) (meta_collector_collector_event_api_server_received{pod=\"$pod\", namespace=\"$namespace\", source=\"api-server\"})",
-          "legendFormat": "__auto",
+          "exemplar": false,
+          "expr": "sum by(kind) (meta_collector_broker_dispatched_events{pod=\"$pod\", namespace=\"$namespace\"})",
+          "format": "time_series",
+          "legendFormat": "{{kind}}",
           "range": true,
           "refId": "A"
         }
       ],
-      "title": "Events From Api Server Per collector",
+      "title": "Events Dispatched Per Resource Kind",
       "type": "bargauge"
     },
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "fieldConfig": {
         "defaults": {
@@ -692,7 +643,7 @@
         "h": 8,
         "w": 12,
         "x": 12,
-        "y": 32
+        "y": 24
       },
       "id": 25,
       "options": {
@@ -707,14 +658,16 @@
           "fields": "",
           "values": false
         },
-        "textMode": "auto"
+        "showPercentChange": false,
+        "textMode": "auto",
+        "wideLayout": true
       },
-      "pluginVersion": "9.5.1",
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "expr": "sum(meta_collector_collector_event_api_server_received{pod=\"$pod\", namespace=\"$namespace\", source=\"api-server\"})",
@@ -729,43 +682,13 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
-      "description": "How long in seconds an item stays in workqueue before being requested",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "drawStyle": "line",
-            "fillOpacity": 10,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "normal"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
+            "fixedColor": "semi-dark-orange",
+            "mode": "fixed"
           },
           "mappings": [],
           "thresholds": {
@@ -774,83 +697,58 @@
               {
                 "color": "green",
                 "value": null
-              },
-              {
-                "color": "red",
-                "value": 80
               }
             ]
-          },
-          "unit": "s"
+          }
         },
         "overrides": []
       },
       "gridPos": {
-        "h": 7,
+        "h": 8,
         "w": 12,
         "x": 0,
-        "y": 40
+        "y": 32
       },
-      "id": 30,
+      "id": 24,
       "options": {
-        "legend": {
+        "displayMode": "gradient",
+        "maxVizHeight": 300,
+        "minVizHeight": 10,
+        "minVizWidth": 0,
+        "namePlacement": "auto",
+        "orientation": "horizontal",
+        "reduceOptions": {
           "calcs": [
-            "max",
-            "mean"
+            "lastNotNull"
           ],
-          "displayMode": "list",
-          "placement": "right",
-          "showLegend": true
+          "fields": "",
+          "values": false
         },
-        "tooltip": {
-          "mode": "single",
-          "sort": "none"
-        }
+        "showUnfilled": true,
+        "sizing": "auto",
+        "valueMode": "color"
       },
+      "pluginVersion": "11.0.0",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
-          "exemplar": true,
-          "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
-          "interval": "",
-          "legendFormat": "P50 {{name}} {{instance}} ",
+          "editorMode": "builder",
+          "expr": "sum by(name) (meta_collector_collector_event_api_server_received{pod=\"$pod\", namespace=\"$namespace\", source=\"api-server\"})",
+          "legendFormat": "__auto",
+          "range": true,
           "refId": "A"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prometheus"
-          },
-          "exemplar": true,
-          "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "P90 {{name}} {{instance}} ",
-          "refId": "B"
-        },
-        {
-          "datasource": {
-            "type": "prometheus",
-            "uid": "prometheus"
-          },
-          "exemplar": true,
-          "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
-          "hide": false,
-          "interval": "",
-          "legendFormat": "P99 {{name}} {{instance}} ",
-          "refId": "C"
         }
       ],
-      "title": "Seconds For Items Stay In Queue (before being requested) (P50, P90, P99)",
-      "type": "timeseries"
+      "title": "Events From Api Server Per collector",
+      "type": "bargauge"
     },
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "description": "How long in seconds an item stays in the broker queue before being processed by the broker.",
       "fieldConfig": {
@@ -859,6 +757,7 @@
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -872,6 +771,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -932,10 +832,10 @@
         ]
       },
       "gridPos": {
-        "h": 7,
+        "h": 8,
         "w": 12,
         "x": 12,
-        "y": 40
+        "y": 32
       },
       "id": 13,
       "options": {
@@ -949,6 +849,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -957,7 +858,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "exemplar": true,
@@ -970,7 +871,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "exemplar": true,
@@ -984,7 +885,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "builder",
           "exemplar": true,
@@ -1002,15 +903,16 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
-      "description": "How long in seconds processing an item from workqueue takes.",
+      "description": "How long in seconds an item stays in workqueue before being requested",
       "fieldConfig": {
         "defaults": {
           "color": {
             "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1024,6 +926,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "linear",
             "lineWidth": 1,
             "pointSize": 5,
@@ -1034,7 +937,7 @@
             "spanNulls": false,
             "stacking": {
               "group": "A",
-              "mode": "none"
+              "mode": "normal"
             },
             "thresholdsStyle": {
               "mode": "off"
@@ -1062,20 +965,21 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 47
+        "y": 40
       },
-      "id": 29,
+      "id": 30,
       "options": {
         "legend": {
           "calcs": [
             "max",
             "mean"
           ],
-          "displayMode": "table",
+          "displayMode": "list",
           "placement": "right",
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1084,10 +988,10 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "exemplar": true,
-          "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
+          "expr": "histogram_quantile(0.50, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
           "interval": "",
           "legendFormat": "P50 {{name}} {{instance}} ",
           "refId": "A"
@@ -1095,10 +999,10 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "exemplar": true,
-          "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
+          "expr": "histogram_quantile(0.90, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
           "hide": false,
           "interval": "",
           "legendFormat": "P90 {{name}} {{instance}} ",
@@ -1107,23 +1011,23 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "exemplar": true,
-          "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
+          "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
           "hide": false,
           "interval": "",
           "legendFormat": "P99 {{name}} {{instance}} ",
           "refId": "C"
         }
       ],
-      "title": "Seconds Processing Items From WorkQueue (P50, P90, P99)",
+      "title": "Seconds For Items Stay In Queue (before being requested) (P50, P90, P99)",
       "type": "timeseries"
     },
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "description": "Total number of retries handled by workqueue",
       "fieldConfig": {
@@ -1132,6 +1036,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1145,6 +1050,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 3,
             "pointSize": 5,
@@ -1183,7 +1089,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 47
+        "y": 40
       },
       "id": 34,
       "options": {
@@ -1194,6 +1100,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1202,12 +1109,14 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
+          "editorMode": "code",
           "exemplar": true,
           "expr": "sum(rate(workqueue_retries_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
           "interval": "",
           "legendFormat": "{{name}} {{instance}} ",
+          "range": true,
           "refId": "A"
         }
       ],
@@ -1217,29 +1126,32 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
+      "description": "How long in seconds processing an item from workqueue takes.",
       "fieldConfig": {
         "defaults": {
           "color": {
-            "mode": "continuous-GrYlRd"
+            "mode": "palette-classic"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
             "axisPlacement": "auto",
             "barAlignment": 0,
             "drawStyle": "line",
-            "fillOpacity": 20,
-            "gradientMode": "scheme",
+            "fillOpacity": 10,
+            "gradientMode": "none",
             "hideFrom": {
               "legend": false,
               "tooltip": false,
               "viz": false
             },
-            "lineInterpolation": "smooth",
-            "lineWidth": 3,
+            "insertNulls": false,
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
             "pointSize": 5,
             "scaleDistribution": {
               "type": "linear"
@@ -1268,7 +1180,7 @@
               }
             ]
           },
-          "unit": "ops"
+          "unit": "s"
         },
         "overrides": []
       },
@@ -1276,42 +1188,69 @@
         "h": 7,
         "w": 12,
         "x": 0,
-        "y": 54
+        "y": 47
       },
-      "id": 33,
+      "id": 29,
       "options": {
         "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
+          "calcs": [
+            "max",
+            "mean"
+          ],
+          "displayMode": "table",
+          "placement": "right",
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
       },
-      "pluginVersion": "8.4.3",
       "targets": [
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "exemplar": true,
-          "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
+          "expr": "histogram_quantile(0.50, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
           "interval": "",
-          "legendFormat": "{{name}} {{instance}}",
+          "legendFormat": "P50 {{name}} {{instance}} ",
           "refId": "A"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_Metrics}"
+          },
+          "exemplar": true,
+          "expr": "histogram_quantile(0.90, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "P90 {{name}} {{instance}} ",
+          "refId": "B"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_Metrics}"
+          },
+          "exemplar": true,
+          "expr": "histogram_quantile(0.99, sum(rate(workqueue_work_duration_seconds_bucket{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name, le))",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "P99 {{name}} {{instance}} ",
+          "refId": "C"
         }
       ],
-      "title": "Work Queue Add Rate",
+      "title": "Seconds Processing Items From WorkQueue (P50, P90, P99)",
       "type": "timeseries"
     },
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
       },
       "description": "Total number of reconciliation errors per controller",
       "fieldConfig": {
@@ -1320,6 +1259,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1333,6 +1273,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 3,
             "pointSize": 5,
@@ -1371,7 +1312,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 54
+        "y": 47
       },
       "id": 32,
       "options": {
@@ -1382,6 +1323,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1390,7 +1332,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "code",
           "exemplar": true,
@@ -1407,7 +1349,104 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "prometheus"
+        "uid": "${DS_Metrics}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "continuous-GrYlRd"
+          },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 20,
+            "gradientMode": "scheme",
+            "hideFrom": {
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "insertNulls": false,
+            "lineInterpolation": "smooth",
+            "lineWidth": 3,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "type": "linear"
+            },
+            "showPoints": "auto",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "ops"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 7,
+        "w": 12,
+        "x": 0,
+        "y": 54
+      },
+      "id": 33,
+      "options": {
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "maxHeight": 600,
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "8.4.3",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_Metrics}"
+          },
+          "exemplar": true,
+          "expr": "sum(rate(workqueue_adds_total{job=\"$job\", namespace=\"$namespace\"}[5m])) by (instance, name)",
+          "interval": "",
+          "legendFormat": "{{name}} {{instance}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Work Queue Add Rate",
+      "type": "timeseries"
+    },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_Metrics}"
       },
       "description": "Total number of reconciliations per controller",
       "fieldConfig": {
@@ -1416,6 +1455,7 @@
             "mode": "continuous-GrYlRd"
           },
           "custom": {
+            "axisBorderShow": false,
             "axisCenteredZero": false,
             "axisColorMode": "text",
             "axisLabel": "",
@@ -1429,6 +1469,7 @@
               "tooltip": false,
               "viz": false
             },
+            "insertNulls": false,
             "lineInterpolation": "smooth",
             "lineWidth": 3,
             "pointSize": 5,
@@ -1467,7 +1508,7 @@
         "h": 7,
         "w": 12,
         "x": 12,
-        "y": 61
+        "y": 54
       },
       "id": 31,
       "options": {
@@ -1478,6 +1519,7 @@
           "showLegend": true
         },
         "tooltip": {
+          "maxHeight": 600,
           "mode": "single",
           "sort": "none"
         }
@@ -1486,7 +1528,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "prometheus"
+            "uid": "${DS_Metrics}"
           },
           "editorMode": "code",
           "exemplar": true,
@@ -1502,20 +1544,32 @@
     }
   ],
   "refresh": "5s",
-  "schemaVersion": 38,
-  "style": "dark",
-  "tags": [],
+  "schemaVersion": 39,
+  "tags": [
+    "falco"
+  ],
   "templating": {
     "list": [
       {
-        "current": {
-          "selected": false,
-          "text": "meta-collector",
-          "value": "meta-collector"
-        },
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Metrics",
+        "multi": false,
+        "name": "DS_Metrics",
+        "options": [],
+        "query": "prometheus",
+        "queryValue": "",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "current": {},
         "datasource": {
           "type": "prometheus",
-          "uid": "prometheus"
+          "uid": "${DS_Metrics}"
         },
         "definition": "label_values(meta_collector_server_subscribers,namespace)",
         "hide": 0,
@@ -1524,33 +1578,31 @@
         "name": "namespace",
         "options": [],
         "query": {
+          "qryType": 1,
           "query": "label_values(meta_collector_server_subscribers,namespace)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
-        "refresh": 2,
+        "refresh": 1,
         "regex": "",
         "skipUrlSync": false,
         "sort": 0,
         "type": "query"
       },
       {
-        "current": {
-          "selected": false,
-          "text": "meta-collector-585d94f758-bpsxm",
-          "value": "meta-collector-585d94f758-bpsxm"
-        },
+        "current": {},
         "datasource": {
           "type": "prometheus",
-          "uid": "prometheus"
+          "uid": "${DS_Metrics}"
         },
-        "definition": "label_values(meta_collector_server_subscribers{namespace=~\"$namespace\"},pod)",
+        "definition": "label_values(meta_collector_server_subscribers{namespace=\"$namespace\"},job)",
         "hide": 0,
         "includeAll": false,
         "multi": false,
-        "name": "pod",
+        "name": "job",
         "options": [],
         "query": {
-          "query": "label_values(meta_collector_server_subscribers{namespace=~\"$namespace\"},pod)",
+          "qryType": 1,
+          "query": "label_values(meta_collector_server_subscribers{namespace=\"$namespace\"},job)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -1560,23 +1612,20 @@
         "type": "query"
       },
       {
-        "current": {
-          "selected": false,
-          "text": "meta-collector",
-          "value": "meta-collector"
-        },
+        "current": {},
         "datasource": {
           "type": "prometheus",
-          "uid": "prometheus"
+          "uid": "${DS_Metrics}"
         },
-        "definition": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"},job)",
+        "definition": "label_values(meta_collector_server_subscribers{namespace=\"$namespace\", job=\"$job\"},pod)",
         "hide": 0,
         "includeAll": false,
         "multi": false,
-        "name": "job",
+        "name": "pod",
         "options": [],
         "query": {
-          "query": "label_values(controller_runtime_reconcile_total{namespace=~\"$namespace\"},job)",
+          "qryType": 1,
+          "query": "label_values(meta_collector_server_subscribers{namespace=\"$namespace\", job=\"$job\"},pod)",
           "refId": "PrometheusVariableQueryEditor-VariableQuery"
         },
         "refresh": 1,
@@ -1591,10 +1640,11 @@
     "from": "now-30m",
     "to": "now"
   },
+  "timeRangeUpdatedDuringEditOrView": false,
   "timepicker": {},
   "timezone": "",
-  "title": "Falco Meta Collector",
-  "uid": "T0NvRcb4z",
-  "version": 1,
+  "title": "Falco / Meta Collector",
+  "uid": "fdr5cuh96bj7kf",
+  "version": 4,
   "weekStart": ""
 }