diff --git a/monitoring/__pycache__/dashboard.cpython-311.pyc b/monitoring/__pycache__/dashboard.cpython-311.pyc index 249313ff..f2deadca 100644 Binary files a/monitoring/__pycache__/dashboard.cpython-311.pyc and b/monitoring/__pycache__/dashboard.cpython-311.pyc differ diff --git a/monitoring/dashboard.json b/monitoring/dashboard.json index de208eda..21776520 100644 --- a/monitoring/dashboard.json +++ b/monitoring/dashboard.json @@ -123,7 +123,7 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, "y": 1 }, @@ -144,13 +144,13 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))", + "expr": "rate(s3_countitems_total_objects_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "Objects processed per second - \"${namespace}\" - ${job}", + "legendFormat": "{{status}}", "metric": "", "refId": "", "step": 10, @@ -162,6 +162,89 @@ "transparent": false, "type": "timeseries" }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "heatmap": {}, + "hideTimeOverride": false, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 3, + "legend": { + "show": false + }, + "links": [], + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{le}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Consolidation duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yAxis": { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + }, { "datasource": "${DS_PROMETHEUS}", "editable": true, @@ -202,18 +285,18 @@ "mode": "absolute", "steps": [] }, - "unit": "s" + "unit": "" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 8, + "x": 16, "y": 1 }, "hideTimeOverride": false, - "id": 3, + "id": 4, "links": [], "maxDataPoints": 100, "options": { @@ -229,20 +312,20 @@ "targets": [ { "datasource": null, - "expr": "sum(rate(count_items_consolidationDuration_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval]))", + "expr": "rate(s3_countitems_total_buckets_count{namespace=\"${namespace}\", job=~\"${job}\"}[$__rate_interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, - "legendFormat": "{{namespace}} - {{job}}", + "legendFormat": "{{status}}", "metric": "", "refId": "", "step": 10, "target": "" } ], - "title": "consolidation duration", + "title": "Rate of the S3 buckets processed", "transformations": [], "transparent": false, "type": "timeseries" diff --git a/monitoring/dashboard.py b/monitoring/dashboard.py index 00f27520..69735c16 100644 --- a/monitoring/dashboard.py +++ b/monitoring/dashboard.py @@ -31,25 +31,41 @@ unit="", targets=[ Target( - expr='sum(rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))', - legendFormat='Objects processed per second - "${namespace}" - ${job}' + expr='rate(s3_countitems_total_objects_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])', + legendFormat="{{status}}", ), ], ) -consolidationDuration = TimeSeries( - title="consolidation duration", +bucketProcessingSpeed = TimeSeries( + title="Rate of the S3 buckets processed", dataSource="${DS_PROMETHEUS}", lineInterpolation="smooth", spanNulls=3*60*1000, - unit=UNITS.SECONDS, + unit="", + targets=[ + Target( + expr='rate(s3_countitems_total_buckets_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval])', + legendFormat="{{status}}", + ), + ], +) + +consolidationDuration = Heatmap( + title="Consolidation duration", + dataSource="${DS_PROMETHEUS}", + dataFormat="tsbuckets", + maxDataPoints=25, + tooltip=Tooltip(show=True, showHistogram=True), + yAxis=YAxis(format=UNITS.SECONDS), + color=HeatmapColor(mode="opacity"), targets=[Target( - expr='sum(rate(count_items_consolidationDuration_count{namespace="${namespace}", job=~"${job}"}[$__rate_interval]))', - legendFormat='{{namespace}} - {{job}}' + expr='sum by(le) (increase(s3_countitems_bucket_merge_duration_seconds_bucket{namespace="${namespace}", job="${job}"}[$__rate_interval]))', + format="heatmap", + legendFormat="{{le}}", )], ) - dashboard = ( Dashboard( title="S3Utils service", @@ -93,7 +109,7 @@ ], panels=layout.column([ RowPanel(title="Count items metrics"), - layout.row([objectProcessingSpeed, consolidationDuration], height=8), + layout.row([objectProcessingSpeed, consolidationDuration, bucketProcessingSpeed], height=8), ]), ) .auto_panel_ids()