From bccc8d465875d026070d665ec7459b93f3939b35 Mon Sep 17 00:00:00 2001
From: Maha Benzekri <maha.benzekri@scality.com>
Date: Fri, 17 May 2024 13:06:03 +0200
Subject: [PATCH] wip

---
 .../__pycache__/dashboard.cpython-311.pyc     | Bin 2952 -> 3480 bytes
 monitoring/dashboard.json                     | 103 ++++++++++++++++--
 monitoring/dashboard.py                       |  34 ++++--
 3 files changed, 118 insertions(+), 19 deletions(-)

diff --git a/monitoring/__pycache__/dashboard.cpython-311.pyc b/monitoring/__pycache__/dashboard.cpython-311.pyc
index 249313ffe8044c5b093f333b68a3faf92c5ed031..f2deadca7ce354ebf719f15c79afe1c7ba7a08d9 100644
GIT binary patch
delta 1065
zcmZuv&rcIs9G_q9l<9OzftHpE3}}(EC`xgYz3Hl;hux?}V$h_S_WLNqwjDC>Ri%XV
z;H5X0%pUOOsk<BJK*F8G|DYHSlew%1FPo?-3Gw8?Hx$gS`2FVl&G*OW`@Wx-_Zt0M
zN#)N_NMfk`GBdaIS>9CQ?C$3I9SH&)gJPh7wE6LDJwX#5!r+5$h|o9+J<wzKHaUiQ
zMXe*W9z@E@z)5cRHCfuH_O&lS)Q+91#-JbKkgx}uJjGM%NRhRZ%>b<g>L;beBwf-2
zXXyGt7%H+yEIAkk6-Gd{N1FnLQ|k!Da&Q*TK?=@&r^mt=)$=fJr%%nXFhTVPm^|tw
z{(mS77pPu@DLZYwWTQWw`OiW>T(U2px?%Uq;0XrBQ4%GJJWSge>peRhnQ4lYG_{Vr
z9I~!)z19@h%?=+&;lQE_SF%j{$DI{!f@Sm83qBvLN$cyFXrziatc<XDUA?Cfq#8>q
zSw?C$qvorH-w?s-s%aDu#t3S8lOo1(CRZ@36`~VV#yMgTt&}^)S}oycz81S~Jr|;N
zw~OH8`o9UvZ9dJp0$w(rxFOHq(lIfLrdCd~u7Ale%NlV-V^u5YWQ~@DvH_^}3|Gr)
zer>#j((1TgDVRvZ-!kaz$d!><L^-HZj=E9FVHp*S3g8YkYnNB8fB3<=D=s6Al(p5G
ze|^1#Ha60{>5(&2p7Xw5I$$Qf*n1RL0*z?19de<v_Vfl#?-+W8U{@rDQ6l=P>s!0_
zRL8EbV3Z8gROwhPoXy-PdI_r-nUD1Xa{0_9I+4v-bH2a*8SnU+olm}BS$4#_DQ^3t
zM|ZH^iLZkXXw&=g_%6S*9T;MTs*^}rZ@Q-9lTLWVi48eQzta_b8&Dd4Cvx1JsH<C|
zBaM29@PO0P=g7T{xvjpx=3Dago;<xTPdie$k@~&*$3{y^?MbP9DdkA9#^l!RmNd2}
zjqOWgPN=uB(B>E=%Q@k=6CG@eg5p15s5|srn-M}{I}%E82Ml$Go@+7Dw#xVvYgYOY
cnSI#mf7C)tExEWS7x(3&bv1ZHSfH2r0?&CMDF6Tf

delta 652
zcmZut&1(};5PxsG`PybTY14dck|x_EHXqhp1o0#yejjY4>Y)fF-Fb$DBpb4chr}QV
zLJxw3cj?hnL=9NX!JFXCV+$VkU(kyw^sck1N{h}g!_55No8Rxf-Plf8c`eHlLHo9L
z{m~C;s6^<cVRo{_l$vUfZ4(F!fdana+w{{YHjChg1z3bYg@PM<0hS;LAqbAcBax7I
z7{@llBA`MP)c;@<V(4*L9w+{}M<I!>!OCgu;Y<yM6nYvmr$It04y5$gVkZPR;mmk-
zL{S#TUs?WI{7=UsELssDJI?XCkX*=*TzD4a*j_F3&q9E|7yLBwyYjg=lTg?o`LmPn
zLV-FeX19^!KSh-n#akEiy$kExHKun=t<^SkW||DN7HexJ(~Tx*=d@I&-z+zn*(&R7
zkitQxUo*D(GjX#R!Xu_puR^&~HJTc9+q0tAuqm)+Io;?sJ6qKbYnWU7r6<Ufjw@^V
zUbuf^=Ky#1?VW=`&$n^@ey3VDk-q)7s<V6iqw|Av6(85mP7K!_N;Uq;y~m$>_U3Qn
z)cz#%K$u*X5>)J2>RtY3zHlSGD_K&?@~v2*gr$TnkMd&Kk^?&rEcuBQEKVJBit~_Q
wI<nK0%*n2)EUCgF!E|J&eG;6;`MP&1hp&Ap?|VxJ-qKfZiI?Q7;!WiI1=}{HumAu6

diff --git a/monitoring/dashboard.json b/monitoring/dashboard.json
index de208eda..21776520 100644
--- a/monitoring/dashboard.json
+++ b/monitoring/dashboard.json
@@ -123,7 +123,7 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
+        "w": 8,
         "x": 0,
         "y": 1
       },
@@ -144,13 +144,13 @@
       "targets": [
         {
           "datasource": null,
-          "expr": "sum(rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))",
+          "expr": "rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])",
           "format": "time_series",
           "hide": false,
           "instant": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "Objects processed per second - \"${namespace}\" - ${job}",
+          "legendFormat": "{{status}}",
           "metric": "",
           "refId": "",
           "step": 10,
@@ -162,6 +162,89 @@
       "transparent": false,
       "type": "timeseries"
     },
+    {
+      "cards": {
+        "cardPadding": null,
+        "cardRound": null
+      },
+      "color": {
+        "cardColor": "#b4ff00",
+        "colorScale": "sqrt",
+        "colorScheme": "interpolateOranges",
+        "exponent": 0.5,
+        "max": null,
+        "min": null,
+        "mode": "opacity"
+      },
+      "dataFormat": "tsbuckets",
+      "datasource": "${DS_PROMETHEUS}",
+      "editable": true,
+      "error": false,
+      "fieldConfig": {
+        "defaults": {
+          "thresholds": {
+            "mode": "absolute",
+            "steps": []
+          }
+        }
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 8,
+        "x": 8,
+        "y": 1
+      },
+      "heatmap": {},
+      "hideTimeOverride": false,
+      "hideZeroBuckets": false,
+      "highlightCards": true,
+      "id": 3,
+      "legend": {
+        "show": false
+      },
+      "links": [],
+      "maxDataPoints": 25,
+      "reverseYBuckets": false,
+      "targets": [
+        {
+          "datasource": null,
+          "expr": "sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))",
+          "format": "heatmap",
+          "hide": false,
+          "instant": false,
+          "interval": "",
+          "intervalFactor": 1,
+          "legendFormat": "{{le}}",
+          "metric": "",
+          "refId": "",
+          "step": 10,
+          "target": ""
+        }
+      ],
+      "title": "Consolidation duration",
+      "tooltip": {
+        "show": true,
+        "showHistogram": true
+      },
+      "transformations": [],
+      "transparent": false,
+      "type": "heatmap",
+      "xAxis": {
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yAxis": {
+        "decimals": null,
+        "format": "s",
+        "label": null,
+        "logBase": 1,
+        "max": null,
+        "min": null,
+        "show": true
+      }
+    },
     {
       "datasource": "${DS_PROMETHEUS}",
       "editable": true,
@@ -202,18 +285,18 @@
             "mode": "absolute",
             "steps": []
           },
-          "unit": "s"
+          "unit": ""
         },
         "overrides": []
       },
       "gridPos": {
         "h": 8,
-        "w": 12,
-        "x": 12,
+        "w": 8,
+        "x": 16,
         "y": 1
       },
       "hideTimeOverride": false,
-      "id": 3,
+      "id": 4,
       "links": [],
       "maxDataPoints": 100,
       "options": {
@@ -229,20 +312,20 @@
       "targets": [
         {
           "datasource": null,
-          "expr": "sum(rate(count_items_consolidationDuration_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))",
+          "expr": "rate(s3_countitems_total_buckets_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])",
           "format": "time_series",
           "hide": false,
           "instant": false,
           "interval": "",
           "intervalFactor": 1,
-          "legendFormat": "{{namespace}} - {{job}}",
+          "legendFormat": "{{status}}",
           "metric": "",
           "refId": "",
           "step": 10,
           "target": ""
         }
       ],
-      "title": "consolidation duration",
+      "title": "Rate of the S3 buckets processed",
       "transformations": [],
       "transparent": false,
       "type": "timeseries"
diff --git a/monitoring/dashboard.py b/monitoring/dashboard.py
index 00f27520..69735c16 100644
--- a/monitoring/dashboard.py
+++ b/monitoring/dashboard.py
@@ -31,25 +31,41 @@
     unit="",
     targets=[
         Target(
-            expr='sum(rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))',
-            legendFormat='Objects processed per second - "${namespace}" - ${job}'
+            expr='rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])',
+            legendFormat="{{status}}",
         ),
     ],
 )
 
-consolidationDuration = TimeSeries(
-    title="consolidation duration",
+bucketProcessingSpeed = TimeSeries(
+    title="Rate of the S3 buckets processed",
     dataSource="${DS_PROMETHEUS}",
     lineInterpolation="smooth",
     spanNulls=3*60*1000,
-    unit=UNITS.SECONDS,
+    unit="",
+    targets=[
+        Target(
+            expr='rate(s3_countitems_total_buckets_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])',
+            legendFormat="{{status}}",
+        ),
+    ],
+)
+
+consolidationDuration = Heatmap(
+    title="Consolidation duration",
+    dataSource="${DS_PROMETHEUS}",
+    dataFormat="tsbuckets",
+    maxDataPoints=25,
+    tooltip=Tooltip(show=True, showHistogram=True),
+    yAxis=YAxis(format=UNITS.SECONDS),
+    color=HeatmapColor(mode="opacity"),
     targets=[Target(
-        expr='sum(rate(count_items_consolidationDuration_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))',
-        legendFormat='{{namespace}} - {{job}}'
+        expr='sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace="${namespace}", job="${job}"}[$__rate_interval]))',
+        format="heatmap",
+        legendFormat="{{le}}",
     )],
 )
 
-
 dashboard = (
     Dashboard(
         title="S3Utils service",
@@ -93,7 +109,7 @@
         ],
         panels=layout.column([
             RowPanel(title="Count items metrics"),
-            layout.row([objectProcessingSpeed, consolidationDuration], height=8),
+            layout.row([objectProcessingSpeed, consolidationDuration, bucketProcessingSpeed], height=8),
         ]),
     )
     .auto_panel_ids()