diff --git a/dashboards/grafana-dashboard-insights-rbac-operations.configmap.yaml b/dashboards/grafana-dashboard-insights-rbac-operations.configmap.yaml index 443bf8e5e..865b6a9bf 100644 --- a/dashboards/grafana-dashboard-insights-rbac-operations.configmap.yaml +++ b/dashboards/grafana-dashboard-insights-rbac-operations.configmap.yaml @@ -73,12 +73,12 @@ data: "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, + "id": 687907, "links": [], "liveNow": false, "panels": [ { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -86,266 +86,267 @@ data: "y": 0 }, "id": 45, - "panels": [], - "title": "RBAC Cache", - "type": "row" - }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 16, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineStyle": { - "fill": "solid" - }, - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } + "panels": [ + { + "datasource": { + "uid": "$datasource" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "yellow", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "color": "orange", - "value": 50 + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 16, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } }, - { - "color": "dark-red", - "value": 80 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "orange", + "value": 50 + }, + { + "color": "dark-red", + "value": 80 + } + ] } - ] - } - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "sum(rate(redis_enable_cache_get_total {job=\"rbac-service\"}[30m]))" }, - "properties": [ + "overrides": [ { - "id": "displayName", - "value": "cache enable" - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "sum(rate(redis_disable_cache_get_total {job=\"rbac-service\"}[30m]))" - }, - "properties": [ + "matcher": { + "id": "byName", + "options": "sum(rate(redis_enable_cache_get_total {job=\"rbac-service\"}[30m]))" + }, + "properties": [ + { + "id": "displayName", + "value": "cache enable" + } + ] + }, { - "id": "displayName", - "value": "cache disable" + "matcher": { + "id": "byName", + "options": "sum(rate(redis_disable_cache_get_total {job=\"rbac-service\"}[30m]))" + }, + "properties": [ + { + "id": "displayName", + "value": "cache disable" + } + ] } ] - } - ] - }, - "gridPos": { - "h": 7, - "w": 9, - "x": 0, - "y": 1 - }, - "id": 47, - "options": { - "legend": { - "calcs": [], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "uid": "$datasource" }, - "editorMode": "code", - "expr": "sum(rate(redis_enable_cache_get_total {job=\"rbac-service\"}[30m]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" + "gridPos": { + "h": 7, + "w": 9, + "x": 0, + "y": 1 }, - "editorMode": "code", - "expr": "sum(rate(redis_disable_cache_get_total {job=\"rbac-service\"}[30m]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - } - ], - "title": " RBAC Cache Request Status ", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "id": 47, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "value": null + "targets": [ + { + "datasource": { + "uid": "$datasource" }, - { - "color": "green", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 9, - "y": 1 - }, - "id": 49, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "editorMode": "code", + "expr": "sum(rate(redis_enable_cache_get_total {job=\"rbac-service\"}[30m]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(redis_disable_cache_get_total {job=\"rbac-service\"}[30m]))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } ], - "fields": "", - "values": false + "title": " RBAC Cache Request Status ", + "type": "timeseries" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.3.8", - "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "editorMode": "code", - "expr": "sum(increase(redis_enable_cache_get_total{job=\"rbac-service\"}[$__range]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Total Cache Enable Requests", - "type": "gauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - { - "color": "red", - "value": 1 + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 12, - "y": 1 - }, - "id": 51, - "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 49, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(increase(redis_enable_cache_get_total{job=\"rbac-service\"}[$__range]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Total Cache Enable Requests", + "type": "gauge" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.3.8", - "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "editorMode": "code", - "expr": "sum(increase(redis_disable_cache_get_total{job=\"rbac-service\"}[$__range]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 12, + "y": 1 + }, + "id": 51, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(increase(redis_disable_cache_get_total{job=\"rbac-service\"}[$__range]))", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total Cache Disable Requests", + "type": "gauge" } ], - "title": "Total Cache Disable Requests", - "type": "gauge" + "title": "RBAC Cache", + "type": "row" }, { "collapsed": true, @@ -356,7 +357,7 @@ data: "h": 1, "w": 24, "x": 0, - "y": 8 + "y": 1 }, "id": 27, "panels": [ @@ -380,17 +381,109 @@ data: "h": 9, "w": 12, "x": 0, - "y": 33 + "y": 2 + }, + "hiddenSeries": false, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.8", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "rate(container_cpu_usage_seconds_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "CPU", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "CPU consumption plotted against configured CPU limit of service", + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 2 }, "hiddenSeries": false, - "id": 11, + "id": 15, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -409,25 +502,39 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "rate(container_cpu_usage_seconds_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m])", - "interval": "", - "legendFormat": "{{pod}}", + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m])) / sum(kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"$Namespace\",pod=~\".*rbac.*\"})", + "interval": "15s", + "legendFormat": "limits", + "range": true, "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$Namespace\", pod=~\".*rbac.*\"}[5m])) / sum(kube_pod_container_resource_requests{resource=\"cpu\", namespace=\"$Namespace\", pod=~\".*rbac.*\"})", + "hide": false, + "interval": "15s", + "legendFormat": "requests", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "CPU", + "title": "Percentage of CPU limit and request consumed", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -438,14 +545,15 @@ data: }, "yaxes": [ { - "format": "s", + "format": "percentunit", "logBase": 1, + "min": "0", "show": true }, { "format": "short", "logBase": 1, - "show": true + "show": false } ], "yaxis": { @@ -460,7 +568,6 @@ data: "datasource": { "uid": "$datasource" }, - "description": "CPU consumption plotted against configured CPU limit of service", "fieldConfig": { "defaults": { "links": [] @@ -472,17 +579,19 @@ data: "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 33 + "x": 0, + "y": 11 }, "hiddenSeries": false, - "id": 15, + "id": 9, "legend": { + "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "show": false, + "rightSide": true, + "show": true, "total": false, "values": false }, @@ -500,25 +609,25 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(container_cpu_usage_seconds_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m])) / sum(kube_pod_container_resource_limits{resource=\"cpu\",namespace=\"$Namespace\",pod=~\".*rbac.*\"})", + "expr": "container_memory_usage_bytes{namespace=\"$Namespace\",pod=~\".*rbac.*\"}", "interval": "15s", - "legendFormat": "", + "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Percentage of CPU limit consumed", + "title": "Memory", "tooltip": { "shared": true, - "sort": 0, + "sort": 2, "value_type": "individual" }, "type": "graph", @@ -529,15 +638,14 @@ data: }, "yaxes": [ { - "format": "percentunit", + "format": "bytes", "logBase": 1, - "min": "0", "show": true }, { "format": "short", "logBase": 1, - "show": false + "show": true } ], "yaxis": { @@ -550,8 +658,10 @@ data: "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, + "description": "", "fieldConfig": { "defaults": { "links": [] @@ -563,18 +673,16 @@ data: "gridPos": { "h": 9, "w": 12, - "x": 0, - "y": 42 + "x": 12, + "y": 11 }, "hiddenSeries": false, - "id": 9, + "id": 19, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false @@ -593,25 +701,39 @@ data: "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "container_memory_usage_bytes{namespace=\"$Namespace\",pod=~\".*rbac.*\"}", + "editorMode": "code", + "expr": "sum(container_memory_usage_bytes{namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"}) / sum(kube_pod_container_resource_limits{resource=\"memory\",namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"})", "interval": "15s", - "legendFormat": "{{pod}}", + "legendFormat": "limits", + "range": true, "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(container_memory_usage_bytes{namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"}) / sum(kube_pod_container_resource_requests{resource=\"memory\",namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"})", + "hide": false, + "interval": "15s", + "legendFormat": "requests", + "range": true, + "refId": "B" } ], "thresholds": [], "timeRegions": [], - "title": "Memory", + "title": "Percentage of memory limit and request consumed", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -622,14 +744,16 @@ data: }, "yaxes": [ { - "format": "bytes", + "decimals": 1, + "format": "percentunit", "logBase": 1, + "max": "1", "show": true }, { "format": "short", "logBase": 1, - "show": true + "show": false } ], "yaxis": { @@ -642,9 +766,10 @@ data: "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "description": "", + "description": "Plots the number of container restarts in 5 minute chunks", "fieldConfig": { "defaults": { "links": [] @@ -656,17 +781,17 @@ data: "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 42 + "x": 0, + "y": 20 }, "hiddenSeries": false, - "id": 19, + "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -691,15 +816,17 @@ data: "datasource": { "uid": "$datasource" }, - "expr": "sum(container_memory_usage_bytes{namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"}) / sum(kube_pod_container_resource_limits{resource=\"memory\",namespace=\"$Namespace\",pod=~\".*rbac.*\",container!~\"sti-build|deployment|lifecycle\",name=\"\"})", + "editorMode": "code", + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m]))", "interval": "15s", - "legendFormat": "", + "legendFormat": "restarts total", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Percentage of memory limit consumed", + "title": "Container Restarts", "tooltip": { "shared": true, "sort": 0, @@ -713,157 +840,243 @@ data: }, "yaxes": [ { - "decimals": 1, - "format": "percentunit", + "decimals": 0, + "format": "short", "logBase": 1, - "max": "1", + "min": "0", "show": true }, { "format": "short", "logBase": 1, - "show": false + "show": true } ], "yaxis": { "align": false } + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PD776AFABBE26000A" + }, + "refId": "A" + } + ], + "title": "Operational Data (CPU/Mem)", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PD776AFABBE26000A" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 23, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 91 + }, + "id": 4, + "interval": "", + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"2..\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "2XX Response Rate", + "type": "gauge" + }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 10 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 91 + }, + "id": 6, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"4..\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "4XX Response Rate", + "type": "gauge" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "uid": "$datasource" }, - "description": "Plots the number of container restarts in 5 minute chunks", + "description": "", "fieldConfig": { "defaults": { - "links": [] + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 5 + } + ] + }, + "unit": "reqps" }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 51 - }, - "hiddenSeries": false, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "h": 5, + "w": 4, + "x": 8, + "y": 91 }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", + "id": 5, "options": { - "alertThreshold": true + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true }, - "percentage": false, "pluginVersion": "9.3.8", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=\"$Namespace\",pod=~\".*rbac.*\"}[5m]))", - "interval": "15s", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"5..\"}[$__rate_interval]))", + "instant": false, + "interval": "", "legendFormat": "", "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Container Restarts", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "decimals": 0, - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PD776AFABBE26000A" - }, - "refId": "A" - } - ], - "title": "Operational Data (CPU/Mem)", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PD776AFABBE26000A" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 9 - }, - "id": 23, - "panels": [ + "title": "5XX Response Rate", + "type": "gauge" + }, { "datasource": { - "type": "prometheus", "uid": "$datasource" }, - "description": "", "fieldConfig": { "defaults": { "mappings": [], - "noValue": "0", "thresholds": { "mode": "percentage", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -874,16 +1087,16 @@ data: "gridPos": { "h": 5, "w": 4, - "x": 0, - "y": 34 + "x": 12, + "y": 91 }, - "id": 4, + "id": 14, "interval": "", "options": { "orientation": "auto", "reduceOptions": { "calcs": [ - "last" + "mean" ], "fields": "", "values": false @@ -897,35 +1110,28 @@ data: "datasource": { "uid": "$datasource" }, - "editorMode": "code", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"2..\"}[$__rate_interval]))", - "instant": false, + "expr": "sum(rate(rbac_req_type_total{behalf=\"principal\"}[$__rate_interval]))", "interval": "", "legendFormat": "", "refId": "A" } ], - "title": "2XX Response Rate", + "title": "Principal Behalf", "type": "gauge" }, { "datasource": { "uid": "$datasource" }, - "description": "", "fieldConfig": { "defaults": { "mappings": [], - "noValue": "0", "thresholds": { - "mode": "absolute", + "mode": "percentage", "steps": [ { - "color": "green" - }, - { - "color": "red", - "value": 10 + "color": "green", + "value": null } ] }, @@ -936,246 +1142,502 @@ data: "gridPos": { "h": 5, "w": 4, - "x": 4, - "y": 34 + "x": 16, + "y": 91 }, - "id": 6, + "id": 25, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ - "lastNotNull" + "mean" ], "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "pluginVersion": "9.3.8", - "targets": [ + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(rbac_req_type_total{behalf=\"system\"}[$__rate_interval]))", + "instant": false, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "title": "System/Service Behalf", + "type": "gauge" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "uid": "$datasource" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 96 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "maxDataPoints": 25, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "9.3.8", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{job=\"rbac-service\"}[$__rate_interval])) by (method, status, view)", + "interval": "", + "legendFormat": "{{method}}: {{view}} -- {{status}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Request Status/method/view", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "show": true + }, { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"4..\"}[$__rate_interval]))", - "instant": false, - "interval": "", - "legendFormat": "", - "refId": "A" + "format": "short", + "logBase": 1, + "show": true } ], - "title": "4XX Response Rate", - "type": "gauge" + "yaxis": { + "align": false + } }, { + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", "datasource": { "uid": "$datasource" }, - "description": "", "fieldConfig": { "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 5 - } - ] - }, - "unit": "reqps" + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 4, - "x": 8, - "y": 34 + "h": 9, + "w": 12, + "x": 12, + "y": 96 }, - "id": 5, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 2, + "interval": "", + "legend": { + "show": false + }, + "maxDataPoints": 25, "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } }, "pluginVersion": "9.3.8", + "reverseYBuckets": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"5..\"}[$__rate_interval]))", - "instant": false, + "expr": "sum(increase(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\"}[$__interval])) by (le)", + "format": "heatmap", "interval": "", - "legendFormat": "", + "legendFormat": "{{le}}", "refId": "A" } ], - "title": "5XX Response Rate", - "type": "gauge" + "title": "Endpoint Latency ($Endpoint)", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" }, { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, "datasource": { "uid": "$datasource" }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "reqps" - }, - "overrides": [] - }, + "fill": 1, + "fillGradient": 0, "gridPos": { - "h": 5, - "w": 4, - "x": 12, - "y": 34 + "h": 8, + "w": 12, + "x": 0, + "y": 105 }, - "id": 14, - "interval": "", + "hiddenSeries": false, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "alertThreshold": true }, + "percentage": false, "pluginVersion": "9.3.8", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(rbac_req_type_total{behalf=\"principal\"}[$__rate_interval]))", + "expr": "sum(rate(bop_request_status_total{}[$__rate_interval])) by (method, status)", "interval": "", - "legendFormat": "", + "legendFormat": "{{method}}: {{status}}", "refId": "A" } ], - "title": "Principal Behalf", - "type": "gauge" + "thresholds": [], + "timeRegions": [], + "title": "BOP Request/Status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": false + } + ], + "yaxis": { + "align": false + } }, { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "mappings": [], - "thresholds": { - "mode": "percentage", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "reqps" + "cards": {}, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 4, - "x": 16, - "y": 34 + "h": 8, + "w": 12, + "x": 12, + "y": 105 }, - "id": 25, + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 12, + "legend": { + "show": false + }, + "maxDataPoints": 25, "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "calculate": false, + "calculation": {}, + "cellGap": 2, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#b4ff00", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlGn", + "steps": 128 }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": false + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "show": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } }, "pluginVersion": "9.3.8", + "reverseYBuckets": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(rbac_req_type_total{behalf=\"system\"}[$__rate_interval]))", - "instant": false, + "expr": "sum(increase(rbac_proxy_request_processing_seconds_bucket{}[$__interval])) by (le)", + "format": "heatmap", "interval": "", - "legendFormat": "", + "legendFormat": "{{le}}", "refId": "A" } ], - "title": "System/Service Behalf", - "type": "gauge" - }, + "title": "RBAC to BOP Latency", + "tooltip": { + "show": true, + "showHistogram": false + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PD776AFABBE26000A" + }, + "refId": "A" + } + ], + "title": "RBAC Internal", + "type": "row" + }, + { + "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "PD776AFABBE26000A" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 21, + "panels": [ { "aliasColors": {}, - "bars": true, + "bars": false, "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 9, "w": 12, "x": 0, - "y": 39 + "y": 4 }, "hiddenSeries": false, - "id": 8, + "id": 3, "legend": { - "alignAsTable": true, "avg": false, - "current": true, + "current": false, "max": false, "min": false, - "rightSide": true, - "show": true, - "sort": "current", - "sortDesc": true, + "show": false, "total": false, - "values": true + "values": false }, - "lines": false, + "lines": true, "linewidth": 1, - "maxDataPoints": 25, + "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "9.3.8", - "pointradius": 2, + "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], @@ -1187,15 +1649,17 @@ data: "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(django_http_responses_total_by_status_view_method_total{job=\"rbac-service\"}[$__rate_interval])) by (method, status, view)", - "interval": "", - "legendFormat": "{{method}}: {{view}} -- {{status}}", + "editorMode": "code", + "expr": "sum(rate(api_3scale_gateway_api_time_count{exported_service=\"rbac\"}[5m]))", + "interval": "15s", + "legendFormat": "", + "range": true, "refId": "A" } ], "thresholds": [], "timeRegions": [], - "title": "Request Status/method/view", + "title": "Request Rate", "tooltip": { "shared": true, "sort": 0, @@ -1210,7 +1674,6 @@ data: "yaxes": [ { "format": "short", - "label": "", "logBase": 1, "show": true }, @@ -1224,14 +1687,109 @@ data: "align": false } }, + { + "datasource": { + "uid": "$datasource" + }, + "description": "Proportion of 5xx status codes returned from this service.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "max": 10, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 13, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "uid": "$datasource" + }, + "expr": "sum(rate(api_3scale_gateway_api_status{exported_service=\"rbac\",status=\"5xx\"}[5m])) / sum(rate(api_3scale_gateway_api_time_count{exported_service=\"rbac\"}[5m]))", + "interval": "15s", + "legendFormat": "", + "refId": "A" + } + ], + "title": "Error Rate", + "type": "timeseries" + }, { "cards": {}, "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" + "cardColor": "#c15c17", + "colorScale": "linear", + "colorScheme": "interpolateOranges", + "exponent": 0.9, + "mode": "opacity" }, "dataFormat": "tsbuckets", "datasource": { @@ -1255,17 +1813,17 @@ data: "gridPos": { "h": 9, "w": 12, - "x": 12, - "y": 39 + "x": 0, + "y": 13 }, "heatmap": {}, "hideZeroBuckets": false, "highlightCards": true, - "id": 2, - "interval": "", + "id": 7, "legend": { - "show": false + "show": true }, + "links": [], "maxDataPoints": 25, "options": { "calculate": false, @@ -1274,11 +1832,11 @@ data: "cellValues": {}, "color": { "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", + "fill": "#c15c17", + "mode": "opacity", "reverse": false, "scale": "exponential", - "scheme": "RdYlGn", + "scheme": "Oranges", "steps": 128 }, "exemplars": { @@ -1288,7 +1846,7 @@ data: "le": 1e-9 }, "legend": { - "show": false + "show": true }, "rowsFrame": { "layout": "auto" @@ -1301,7 +1859,7 @@ data: "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "s" + "unit": "dtdurations" } }, "pluginVersion": "9.3.8", @@ -1311,14 +1869,14 @@ data: "datasource": { "uid": "$datasource" }, - "expr": "sum(increase(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\"}[$__interval])) by (le)", + "expr": "sum(increase(api_3scale_gateway_api_time_bucket{exported_service=\"rbac\"}[$__interval])) by (le)", "format": "heatmap", - "interval": "", - "legendFormat": "{{le}}", + "interval": "15s", + "legendFormat": "{{le}} ms", "refId": "A" } ], - "title": "Endpoint Latency ($Endpoint)", + "title": "Latency", "tooltip": { "show": true, "showHistogram": false @@ -1328,70 +1886,335 @@ data: "show": true }, "yAxis": { - "format": "s", + "format": "dtdurations", "logBase": 1, "show": true }, "yBucketBound": "auto" + } + ], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PD776AFABBE26000A" + }, + "refId": "A" + } + ], + "title": "RBAC from Gateway", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 29, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 5 + }, + "id": 39, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "\n\nThe percentage of time the RBAC has been up (as proportion of the 5xx requests out of the total number of requests) during last 28 days > 99 %", + "mode": "markdown" + }, + "pluginVersion": "9.3.8", + "title": "RBAC Availability SLO", + "type": "text" }, { - "aliasColors": {}, - "bars": true, + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 3, + "y": 5 + }, + "id": 31, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "avg_over_time(service:sli:status_5xx:pctl5rate5m{exported_service=\"rbac\"}[28d])", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Error Rate Uptime [28d]", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "min": 0.96, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 7, + "y": 5 + }, + "id": 33, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg_over_time(service:sli:status_5xx:pctl5rate5m{exported_service=\"rbac\"}[5m])", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Error Rate Uptime", + "range": true, + "refId": "A" + } + ], + "title": "Error Rate Uptime", + "type": "timeseries" + }, + { + "aliasColors": { + "1000 - 1500ms": "dark-yellow", + "1500 - 2000ms": "light-yellow", + "2000 - 2500ms": "light-orange", + "2500 - 3000ms": "dark-orange", + "3000 - 3500ms": "light-red", + "3500 - 4000ms": "semi-dark-red", + "500 - 1000ms": "light-green", + "> 4000ms": "dark-red" + }, + "bars": false, "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, + "fieldConfig": { + "defaults": { + "links": [] + }, + "overrides": [] + }, "fill": 1, - "fillGradient": 0, + "fillGradient": 6, "gridPos": { - "h": 8, - "w": 12, + "h": 10, + "w": 20, "x": 0, - "y": 48 + "y": 12 }, "hiddenSeries": false, - "id": 16, + "id": 42, "legend": { - "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, - "rightSide": true, "show": true, "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, + "percentage": true, "pluginVersion": "9.3.8", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(bop_request_status_total{}[$__rate_interval])) by (method, status)", - "interval": "", - "legendFormat": "{{method}}: {{status}}", + "editorMode": "code", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"2..\"}[5m])) > 0", + "legendFormat": "2xx", + "range": true, "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"3..\"}[5m])) > 0", + "hide": false, + "legendFormat": "3xx", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"4..\"}[5m])) > 0", + "hide": false, + "legendFormat": "4xx", + "range": true, + "refId": "C" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"5..\"}[5m])) > 0", + "hide": false, + "legendFormat": "5xx", + "range": true, + "refId": "D" } ], "thresholds": [], "timeRegions": [], - "title": "BOP Request/Status", + "title": "Status Code Proportions", "tooltip": { "shared": true, "sort": 0, @@ -1405,11 +2228,15 @@ data: }, "yaxes": [ { + "$$hashKey": "object:177", "format": "short", "logBase": 1, + "max": "100", + "min": "99", "show": true }, { + "$$hashKey": "object:178", "format": "short", "logBase": 1, "show": false @@ -1420,147 +2247,215 @@ data: } }, { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateRdYlGn", - "exponent": 0.5, - "mode": "spectrum" + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "dataFormat": "tsbuckets", + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 22 + }, + "id": 38, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "The proportion of the service request durations under 500 ms during 28 days > 99 %", + "mode": "markdown" + }, + "pluginVersion": "9.3.8", + "title": "RBAC Latency SLO", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 3, + "y": 22 + }, + "id": 40, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"$Namespace\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[28d]))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Latency Uptime [28d]", + "type": "gauge" + }, + { "datasource": { + "type": "prometheus", "uid": "$datasource" }, + "description": "Alert: Request with duration under 500ms > 95%", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" } - } + }, + "links": [], + "mappings": [], + "min": 0.9, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + }, + { + "color": "red", + "value": 0.99 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 12, - "legend": { - "show": false + "h": 7, + "w": 13, + "x": 7, + "y": 22 }, - "maxDataPoints": 25, + "id": 41, "options": { - "calculate": false, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "RdYlGn", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "show": true, - "yHistogram": false + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "s" + "tooltip": { + "mode": "multi", + "sort": "none" } }, "pluginVersion": "9.3.8", - "reverseYBuckets": false, "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "expr": "sum(increase(rbac_proxy_request_processing_seconds_bucket{}[$__interval])) by (le)", + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"$Namespace\"}[5m])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[5m]))", "format": "heatmap", + "hide": false, + "instant": false, "interval": "", - "legendFormat": "{{le}}", + "legendFormat": "requests < 500 ms", + "range": true, "refId": "A" } ], - "title": "RBAC to BOP Latency", - "tooltip": { - "show": true, - "showHistogram": false - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "s", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - } - ], - "targets": [ + "title": "Latency Uptime", + "type": "timeseries" + }, { - "datasource": { - "type": "prometheus", - "uid": "PD776AFABBE26000A" + "aliasColors": { + "1000 - 1500ms": "dark-yellow", + "1500 - 2000ms": "light-yellow", + "2000 - 2500ms": "light-orange", + "2500 - 3000ms": "dark-orange", + "3000 - 3500ms": "light-red", + "3500 - 4000ms": "semi-dark-red", + "500 - 1000ms": "light-green", + "> 4000ms": "dark-red" }, - "refId": "A" - } - ], - "title": "RBAC Internal", - "type": "row" - }, - { - "collapsed": true, - "datasource": { - "type": "prometheus", - "uid": "PD776AFABBE26000A" - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 10 - }, - "id": 21, - "panels": [ - { - "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, "fieldConfig": { @@ -1570,54 +2465,85 @@ data: "overrides": [] }, "fill": 1, - "fillGradient": 0, + "fillGradient": 6, "gridPos": { - "h": 9, - "w": 12, + "h": 10, + "w": 20, "x": 0, - "y": 35 + "y": 29 }, "hiddenSeries": false, - "id": 3, + "id": 43, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, - "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, - "percentage": false, + "percentage": true, "pluginVersion": "9.3.8", - "pointradius": 5, + "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, - "stack": false, + "stack": true, "steppedLine": false, "targets": [ { "datasource": { "uid": "$datasource" }, - "expr": "sum(rate(api_3scale_gateway_api_time_count{exported_service=\"rbac\"}[5m]))", - "interval": "15s", - "legendFormat": "", + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"$Namespace\"}[5m])) > 0", + "legendFormat": "< 500 ms", + "range": true, "refId": "A" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"1.0\", namespace=\"$Namespace\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"$Namespace\"}[5m])) > 0", + "legendFormat": "500 - 1000 ms", + "range": true, + "refId": "B" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"2.5\", namespace=\"$Namespace\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"1.0\", namespace=\"$Namespace\"}[5m])) > 0", + "legendFormat": "1000 - 2500 ms", + "range": true, + "refId": "E" + }, + { + "datasource": { + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"+Inf\", namespace=\"$Namespace\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"2.5\", namespace=\"$Namespace\"}[5m])) > 0", + "hide": false, + "legendFormat": "> 2500 ms", + "range": true, + "refId": "C" } ], "thresholds": [], "timeRegions": [], - "title": "Request Rate", + "title": "Latency Distribution", "tooltip": { "shared": true, "sort": 0, @@ -1631,14 +2557,18 @@ data: }, "yaxes": [ { + "$$hashKey": "object:177", "format": "short", "logBase": 1, + "max": "100", + "min": "99", "show": true }, { + "$$hashKey": "object:178", "format": "short", "logBase": 1, - "show": true + "show": false } ], "yaxis": { @@ -1646,110 +2576,191 @@ data: } }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "description": "Proportion of 5xx status codes returned from this service.", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 0.5 + } + ] + } }, "overrides": [] }, - "fill": 1, - "fillGradient": 0, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 35 + "h": 6, + "w": 7, + "x": 0, + "y": 39 }, - "hiddenSeries": false, - "id": 13, - "legend": { - "avg": false, - "current": false, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false + "id": 53, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{namespace=\"$Namespace\"}[28d])) by (le))", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + } + ], + "title": "Requests duration 99 % quantile [28d]", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 3, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 13, + "x": 7, + "y": 39 }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null as zero", + "id": 55, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "percentage": false, "pluginVersion": "9.3.8", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, - "expr": "sum(rate(api_3scale_gateway_api_status{exported_service=\"rbac\",status=\"5xx\"}[5m])) / sum(rate(api_3scale_gateway_api_time_count{exported_service=\"rbac\"}[5m]))", - "interval": "15s", - "legendFormat": "", + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{namespace=\"$Namespace\"}[5m])) by (le))", + "legendFormat": "__auto", + "range": true, "refId": "A" } ], - "thresholds": [], - "timeRegions": [], - "title": "Error Rate", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percent", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } - }, + "title": "Requests duration 99 % quantile", + "type": "timeseries" + } + ], + "title": "RBAC SLOs", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 57, + "panels": [ { "cards": {}, "color": { - "cardColor": "#c15c17", - "colorScale": "linear", - "colorScheme": "interpolateOranges", - "exponent": 0.9, - "mode": "opacity" + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateRdYlGn", + "exponent": 0.5, + "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": { + "type": "prometheus", "uid": "$datasource" }, "fieldConfig": { @@ -1771,16 +2782,16 @@ data: "h": 9, "w": 12, "x": 0, - "y": 44 + "y": 6 }, "heatmap": {}, "hideZeroBuckets": false, "highlightCards": true, - "id": 7, + "id": 59, + "interval": "", "legend": { - "show": true + "show": false }, - "links": [], "maxDataPoints": 25, "options": { "calculate": false, @@ -1789,11 +2800,11 @@ data: "cellValues": {}, "color": { "exponent": 0.5, - "fill": "#c15c17", - "mode": "opacity", + "fill": "#b4ff00", + "mode": "scheme", "reverse": false, "scale": "exponential", - "scheme": "Oranges", + "scheme": "RdYlGn", "steps": 128 }, "exemplars": { @@ -1803,7 +2814,7 @@ data: "le": 1e-9 }, "legend": { - "show": true + "show": false }, "rowsFrame": { "layout": "auto" @@ -1816,7 +2827,7 @@ data: "yAxis": { "axisPlacement": "left", "reverse": false, - "unit": "dtdurations" + "unit": "s" } }, "pluginVersion": "9.3.8", @@ -1826,14 +2837,16 @@ data: "datasource": { "uid": "$datasource" }, - "expr": "sum(increase(api_3scale_gateway_api_time_bucket{exported_service=\"rbac\"}[$__interval])) by (le)", + "editorMode": "code", + "expr": "sum(increase(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\"}[$__interval])) by (le)", "format": "heatmap", - "interval": "15s", - "legendFormat": "{{le}} ms", + "interval": "", + "legendFormat": "{{le}}", + "range": true, "refId": "A" } ], - "title": "Latency", + "title": "Endpoint <$Endpoint> Latency", "tooltip": { "show": true, "showHistogram": false @@ -1843,59 +2856,11 @@ data: "show": true }, "yAxis": { - "format": "dtdurations", + "format": "s", "logBase": 1, "show": true }, "yBucketBound": "auto" - } - ], - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PD776AFABBE26000A" - }, - "refId": "A" - } - ], - "title": "RBAC from Gateway", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 11 - }, - "id": 29, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 0, - "y": 36 - }, - "id": 39, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "\n\nThe percentage of time the RBAC has been up (as proportion of the 5xx requests out of the total number of requests) during last 28 days > 99 %", - "mode": "markdown" - }, - "pluginVersion": "9.3.8", - "title": "RBAC Availability SLO", - "type": "text" }, { "datasource": { @@ -1905,20 +2870,49 @@ data: "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 32, + "gradientMode": "hue", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } }, + "links": [], "mappings": [], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ - { - "color": "red" - }, { "color": "green", - "value": 0.99 + "value": null } ] }, @@ -1927,23 +2921,23 @@ data: "overrides": [] }, "gridPos": { - "h": 7, - "w": 4, - "x": 3, - "y": 36 + "h": 9, + "w": 12, + "x": 12, + "y": 6 }, - "id": 31, + "id": 61, "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "tooltip": { + "mode": "single", + "sort": "none" + } }, "pluginVersion": "9.3.8", "targets": [ @@ -1953,22 +2947,73 @@ data: "uid": "$datasource" }, "editorMode": "code", - "expr": "avg_over_time(service:sli:status_5xx:pctl5rate5m{environment=\"prod\", exported_service=\"rbac\"}[28d])", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"0.5\"}[6h])) > 0", + "hide": false, + "legendFormat": "< 500 ms", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"1.0\"}[6h])) - sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"0.5\"}[6h])) > 0", + "hide": false, + "interval": "", + "legendFormat": "500 - 1000 ms", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"2.5\"}[6h])) - sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"1.0\"}[6h])) > 0", + "hide": false, + "interval": "", + "legendFormat": "1000 - 2500 ms", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"5.0\"}[6h])) - sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"2.5\"}[6h])) > 0", + "hide": false, + "interval": "", + "legendFormat": "2500 - 5000 ms", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[6h])) - sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"5.0\"}[6h])) > 0", "hide": false, - "legendFormat": "__auto", + "legendFormat": "> 5000 ms", "range": true, "refId": "B" } ], - "title": "Error Rate Uptime [28d]", - "type": "gauge" + "title": "Latency Distribution for <$Endpoint> endpoint", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, - "description": "", + "description": "Alert: Request with duration under 500ms > 95%", "fieldConfig": { "defaults": { "color": { @@ -2006,16 +3051,12 @@ data: }, "links": [], "mappings": [], - "min": 0.96, "thresholds": { "mode": "absolute", "steps": [ { - "color": "transparent" - }, - { - "color": "red", - "value": 0.99 + "color": "transparent", + "value": null } ] }, @@ -2025,11 +3066,11 @@ data: }, "gridPos": { "h": 7, - "w": 13, - "x": 7, - "y": 36 + "w": 12, + "x": 0, + "y": 15 }, - "id": 33, + "id": 63, "options": { "legend": { "calcs": [], @@ -2050,204 +3091,158 @@ data: "uid": "$datasource" }, "editorMode": "code", - "exemplar": false, - "expr": "avg_over_time(service:sli:status_5xx:pctl5rate5m{environment=\"prod\", exported_service=\"rbac\"}[5m])", - "format": "heatmap", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"0.5\"}[6h])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[6h]))", "hide": false, - "instant": false, - "interval": "", - "legendFormat": "Error Rate Uptime", + "legendFormat": "requests < 500 ms", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "Error Rate Uptime", + "title": "Latency <$Endpoint> endpoint - requests < 500 ms", "type": "timeseries" }, { - "aliasColors": { - "1000 - 1500ms": "dark-yellow", - "1500 - 2000ms": "light-yellow", - "2000 - 2500ms": "light-orange", - "2500 - 3000ms": "dark-orange", - "3000 - 3500ms": "light-red", - "3500 - 4000ms": "semi-dark-red", - "500 - 1000ms": "light-green", - "> 4000ms": "dark-red" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, + "description": "Alert: Request with duration under 500ms > 95%", "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "transparent", + "value": null + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 6, "gridPos": { - "h": 10, - "w": 20, - "x": 0, - "y": 43 - }, - "hiddenSeries": false, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "h": 7, + "w": 12, + "x": 12, + "y": 15 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 65, "options": { - "alertThreshold": true + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "percentage": true, "pluginVersion": "9.3.8", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"2..\"}[5m])) > 0", - "legendFormat": "2xx", - "range": true, - "refId": "A" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"3..\"}[5m])) > 0", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"1.0\"}[6h])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[6h]))", "hide": false, - "legendFormat": "3xx", + "legendFormat": "requests < 1000 ms", "range": true, "refId": "B" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"4..\"}[5m])) > 0", - "hide": false, - "legendFormat": "4xx", - "range": true, - "refId": "C" - }, - { - "datasource": { - "uid": "$datasource" - }, - "editorMode": "code", - "expr": "sum(rate(django_http_responses_total_by_status_total{job=\"rbac-service\", status=~\"5..\"}[5m])) > 0", - "hide": false, - "legendFormat": "5xx", - "range": true, - "refId": "D" - } - ], - "thresholds": [], - "timeRegions": [], - "title": "Status Code Proportions", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:177", - "format": "short", - "logBase": 1, - "max": "100", - "min": "99", - "show": true - }, - { - "$$hashKey": "object:178", - "format": "short", - "logBase": 1, - "show": false } ], - "yaxis": { - "align": false - } - }, - { - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "gridPos": { - "h": 7, - "w": 3, - "x": 0, - "y": 53 - }, - "id": 38, - "options": { - "code": { - "language": "plaintext", - "showLineNumbers": false, - "showMiniMap": false - }, - "content": "The proportion of the service request durations under 500 ms during 28 days > 99 %", - "mode": "markdown" - }, - "pluginVersion": "9.3.8", - "title": "RBAC Latency SLO", - "type": "text" + "title": "Latency $Endpoint endpoint - requests < 1000 ms", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "$datasource" }, + "description": "Alert: Request with duration under 500ms > 95%", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } }, + "links": [], "mappings": [], - "max": 1, - "min": 0, "thresholds": { "mode": "absolute", "steps": [ { - "color": "red" - }, - { - "color": "green", - "value": 0.99 + "color": "transparent", + "value": null } ] }, @@ -2257,22 +3252,22 @@ data: }, "gridPos": { "h": 7, - "w": 4, - "x": 3, - "y": 53 + "w": 12, + "x": 0, + "y": 22 }, - "id": 40, + "id": 67, "options": { - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, - "showThresholdLabels": false, - "showThresholdMarkers": true + "tooltip": { + "mode": "multi", + "sort": "none" + } }, "pluginVersion": "9.3.8", "targets": [ @@ -2282,15 +3277,15 @@ data: "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"rbac-prod\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"rbac-prod\"}[28d]))", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"2.5\"}[6h])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[6h]))", "hide": false, - "legendFormat": "__auto", + "legendFormat": "requests < 2500 ms", "range": true, "refId": "B" } ], - "title": "Latency Uptime [28d]", - "type": "gauge" + "title": "Latency <$Endpoint> endpoint - requests < 2500 ms", + "type": "timeseries" }, { "datasource": { @@ -2335,16 +3330,12 @@ data: }, "links": [], "mappings": [], - "min": 0.9, "thresholds": { "mode": "absolute", "steps": [ { - "color": "transparent" - }, - { - "color": "red", - "value": 0.99 + "color": "transparent", + "value": null } ] }, @@ -2354,11 +3345,11 @@ data: }, "gridPos": { "h": 7, - "w": 13, - "x": 7, - "y": 53 + "w": 12, + "x": 12, + "y": 22 }, - "id": 41, + "id": 69, "options": { "legend": { "calcs": [], @@ -2379,157 +3370,224 @@ data: "uid": "$datasource" }, "editorMode": "code", - "exemplar": false, - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"rbac-prod\"}[5m])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"rbac-prod\"}[5m]))", - "format": "heatmap", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"5.0\"}[6h])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[6h]))", "hide": false, - "instant": false, - "interval": "", - "legendFormat": "requests < 500 ms", + "legendFormat": "requests < 5000 ms", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "Latency Uptime", + "title": "Latency $Endpoint endpoint - requests < 5000 ms", "type": "timeseries" }, { - "aliasColors": { - "1000 - 1500ms": "dark-yellow", - "1500 - 2000ms": "light-yellow", - "2000 - 2500ms": "light-orange", - "2500 - 3000ms": "dark-orange", - "3000 - 3500ms": "light-red", - "3500 - 4000ms": "semi-dark-red", - "500 - 1000ms": "light-green", - "> 4000ms": "dark-red" - }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "$datasource" }, "fieldConfig": { "defaults": { - "links": [] + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" }, "overrides": [] }, - "fill": 1, - "fillGradient": 6, "gridPos": { - "h": 10, - "w": 20, + "h": 7, + "w": 12, "x": 0, - "y": 60 - }, - "hiddenSeries": false, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "y": 29 }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", + "id": 71, "options": { - "alertThreshold": true + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true }, - "percentage": true, "pluginVersion": "9.3.8", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": true, - "steppedLine": false, "targets": [ { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"rbac-prod\"}[5m])) > 0", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"0.5\"}[28d])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[28d]))", + "hide": false, "legendFormat": "< 500 ms", "range": true, "refId": "A" }, { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"1.0\", namespace=\"rbac-prod\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"rbac-prod\"}[5m])) > 0", - "legendFormat": "500 - 1000 ms", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"1.0\"}[28d])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[28d]))", + "hide": false, + "legendFormat": "< 1000 ms", "range": true, - "refId": "B" + "refId": "C" }, { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"2.5\", namespace=\"rbac-prod\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"1.0\", namespace=\"rbac-prod\"}[5m])) > 0", - "legendFormat": "1000 - 2500 ms", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"2.5\"}[28d])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[28d]))", + "hide": false, + "legendFormat": "< 2500 ms", "range": true, - "refId": "E" + "refId": "D" }, { "datasource": { + "type": "prometheus", "uid": "$datasource" }, "editorMode": "code", - "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"+Inf\", namespace=\"rbac-prod\"}[5m])) - sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"2.5\", namespace=\"rbac-prod\"}[5m])) > 0", + "expr": "sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"5.0\"}[28d])) / sum(rate(django_http_requests_latency_seconds_by_view_method_bucket{job=\"rbac-service\", view=~\"$Endpoint\", le=\"+Inf\"}[28d]))", "hide": false, - "legendFormat": "> 2500 ms", + "legendFormat": "< 5000 ms", "range": true, - "refId": "C" + "refId": "B" } ], - "thresholds": [], - "timeRegions": [], - "title": "Latency Distribution", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "title": "Latency Uptime [28d] for <$Endpoint> endpoint", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "yaxes": [ + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 73, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.8", + "targets": [ { - "$$hashKey": "object:177", - "format": "short", - "logBase": 1, - "max": "100", - "min": "99", - "show": true + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"0.5\", namespace=\"$Namespace\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[28d]))", + "hide": false, + "legendFormat": "< 500 ms", + "range": true, + "refId": "A" }, { - "$$hashKey": "object:178", - "format": "short", - "logBase": 1, - "show": false + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"1.0\", namespace=\"$Namespace\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[28d]))", + "hide": false, + "legendFormat": "< 1000 ms", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"2.5\", namespace=\"$Namespace\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[28d]))", + "hide": false, + "legendFormat": "< 2500 ms", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "$datasource" + }, + "editorMode": "code", + "expr": "sum(rate(django_http_requests_latency_including_middlewares_seconds_bucket{le=\"5.0\", namespace=\"$Namespace\"}[28d])) / sum(rate(django_http_requests_latency_including_middlewares_seconds_count{namespace=\"$Namespace\"}[28d]))", + "hide": false, + "legendFormat": "< 5000 ms", + "range": true, + "refId": "D" } ], - "yaxis": { - "align": false - } + "title": "Latency Uptime [28d] for all endpoints", + "type": "gauge" } ], - "title": "RBAC SLOs", + "title": "Endpoints specific metrics (for <$Endpoint>): Please select a endpoint form list in the top of the page", "type": "row" } ], @@ -2559,7 +3617,15 @@ data: "type": "datasource" }, { - "current": {}, + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, "datasource": { "type": "prometheus", "uid": "$datasource" @@ -2584,7 +3650,11 @@ data: "useTags": false }, { - "current": {}, + "current": { + "selected": false, + "text": "rbac-prod", + "value": "rbac-prod" + }, "datasource": { "type": "prometheus", "uid": "$datasource" @@ -2642,7 +3712,7 @@ data: "timezone": "", "title": "Operations - Rbac w/CPU", "uid": "TjP_nMWMk", - "version": 5, + "version": 6, "weekStart": "" } kind: ConfigMap