wip

scality · May 17, 2024 · bccc8d4 · bccc8d4
1 parent 213d1f7
commit bccc8d4
Show file tree

Hide file tree

Showing 3 changed files with 118 additions and 19 deletions.
diff --git a/monitoring/__pycache__/dashboard.cpython-311.pyc b/monitoring/__pycache__/dashboard.cpython-311.pyc
diff --git a/monitoring/dashboard.json b/monitoring/dashboard.json
@@ -123,7 +123,7 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 8,
         "x": 0,
         "y": 1
       },
@@ -144,13 +144,13 @@
       "targets": [
         {
           "datasource": null,
-          "expr": "sum(rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))",
+          "expr": "rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])",
           "format": "time_series",
           "hide": false,
           "instant": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "Objects processed per second - \"${namespace}\" - ${job}",
+          "legendFormat": "{{status}}",
           "metric": "",
           "refId": "",
           "step": 10,
@@ -162,6 +162,89 @@
       "transparent": false,
       "type": "timeseries"
     },
+    {
+      "cards": {
+        "cardPadding": null,
+        "cardRound": null
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateOranges",
+        "exponent": 0.5,
+        "max": null,
+        "min": null,
+        "mode": "opacity"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": "${DS_PROMETHEUS}",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 1
+      },
+      "heatmap": {},
+      "hideTimeOverride": false,
+      "hideZeroBuckets": false,
+      "highlightCards": true,
+      "id": 3,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "maxDataPoints": 25,
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": null,
+          "expr": "sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))",
+          "format": "heatmap",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "metric": "",
+          "refId": "",
+          "step": 10,
+          "target": ""
+        }
+      ],
+      "title": "Consolidation duration",
+      "tooltip": {
+        "show": true,
+        "showHistogram": true
+      },
+      "transformations": [],
+      "transparent": false,
+      "type": "heatmap",
+      "xAxis": {
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yAxis": {
+        "decimals": null,
+        "format": "s",
+        "label": null,
+        "logBase": 1,
+        "max": null,
+        "min": null,
+        "show": true
+      }
+    },
     {
       "datasource": "${DS_PROMETHEUS}",
       "editable": true,
@@ -202,18 +285,18 @@
             "mode": "absolute",
             "steps": []
           },
-          "unit": "s"
+          "unit": ""
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
-        "x": 12,
+        "w": 8,
+        "x": 16,
         "y": 1
       },
       "hideTimeOverride": false,
-      "id": 3,
+      "id": 4,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -229,20 +312,20 @@
       "targets": [
         {
           "datasource": null,
-          "expr": "sum(rate(count_items_consolidationDuration_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))",
+          "expr": "rate(s3_countitems_total_buckets_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])",
           "format": "time_series",
           "hide": false,
           "instant": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{namespace}} - {{job}}",
+          "legendFormat": "{{status}}",
           "metric": "",
           "refId": "",
           "step": 10,
           "target": ""
         }
       ],
-      "title": "consolidation duration",
+      "title": "Rate of the S3 buckets processed",
       "transformations": [],
       "transparent": false,
       "type": "timeseries"

diff --git a/monitoring/dashboard.py b/monitoring/dashboard.py
@@ -31,25 +31,41 @@
     unit="",
     targets=[
         Target(
-            expr='sum(rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))',
-            legendFormat='Objects processed per second - "${namespace}" - ${job}'
+            expr='rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])',
+            legendFormat="{{status}}",
         ),
     ],
 )
 
-consolidationDuration = TimeSeries(
-    title="consolidation duration",
+bucketProcessingSpeed = TimeSeries(
+    title="Rate of the S3 buckets processed",
     dataSource="${DS_PROMETHEUS}",
     lineInterpolation="smooth",
     spanNulls=3*60*1000,
-    unit=UNITS.SECONDS,
+    unit="",
+    targets=[
+        Target(
+            expr='rate(s3_countitems_total_buckets_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])',
+            legendFormat="{{status}}",
+        ),
+    ],
+)
+
+consolidationDuration = Heatmap(
+    title="Consolidation duration",
+    dataSource="${DS_PROMETHEUS}",
+    dataFormat="tsbuckets",
+    maxDataPoints=25,
+    tooltip=Tooltip(show=True, showHistogram=True),
+    yAxis=YAxis(format=UNITS.SECONDS),
+    color=HeatmapColor(mode="opacity"),
     targets=[Target(
-        expr='sum(rate(count_items_consolidationDuration_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))',
-        legendFormat='{{namespace}} - {{job}}'
+        expr='sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace="${namespace}", job="${job}"}[$__rate_interval]))',
+        format="heatmap",
+        legendFormat="{{le}}",
     )],
 )
 
-
 dashboard = (
     Dashboard(
         title="S3Utils service",
@@ -93,7 +109,7 @@
         ],
         panels=layout.column([
             RowPanel(title="Count items metrics"),
-            layout.row([objectProcessingSpeed, consolidationDuration], height=8),
+            layout.row([objectProcessingSpeed, consolidationDuration, bucketProcessingSpeed], height=8),
         ]),
     )
     .auto_panel_ids()