diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b713f075..81746c4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ * improve loading times for the rule tree by optimizing the rule segmentation and sorting * add support for python 3.12 and remove support for python 3.9 * always check the existence of a field for negated key-value based lucene filter expressions +* add kafka exporter to quickstart setup ### Bugfix diff --git a/README.md b/README.md index 6be7777fb..cc23b1f8b 100644 --- a/README.md +++ b/README.md @@ -430,17 +430,18 @@ The start up takes a few seconds to complete, but once everything is up and running it is possible to write JSON events into Kafka and read the processed events in Opensearch Dashboards. Following services are available after start up: -| Service | Location | Credentials | -|:----------|:----|:-----| -| Kafka: | `localhost:9092` | | -| Logprep metrics: | `localhost:8000` | | -| Opensearch: | `localhost:9200` | | +| Service | Location | Credentials | +|:-----------------------|:----|:-----| +| Kafka: | `localhost:9092` | | +| Kafka Exporter: | `localhost:9308` | | +| Logprep metrics: | `localhost:8000` | | +| Opensearch: | `localhost:9200` | | | Opensearch Dashboards: | `localhost:5601` | | -| Grafana Dashboards: | `localhost:3000` | admin:admin | -| Prometheus: | `localhost:9090` | | -| Nginx: | `localhost:8081` | user:password | -| Keycloak: | `localhost:8080` | admin:admin | -| Postgres: | `localhost:5432` | keycloak:bitnami | +| Grafana Dashboards: | `localhost:3000` | admin:admin | +| Prometheus: | `localhost:9090` | | +| Nginx: | `localhost:8081` | user:password | +| Keycloak: | `localhost:8080` | admin:admin | +| Postgres: | `localhost:5432` | keycloak:bitnami | The example rules that are used in the docker instance of Logprep can be found in `quickstart/exampledata/rules`. diff --git a/quickstart/docker-compose.yml b/quickstart/docker-compose.yml index 26380459f..d8cac93b2 100644 --- a/quickstart/docker-compose.yml +++ b/quickstart/docker-compose.yml @@ -61,6 +61,23 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock command: sh -c "((sleep 15 && echo 'kafka up' && kafka-topics.sh --create --if-not-exists --bootstrap-server 127.0.0.1:9092 --replication-factor 1 --partitions 3 --topic consumer)&) && /opt/bitnami/scripts/kafka/run.sh" + healthcheck: + test: + [ + "CMD-SHELL", + "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --topic consumer --describe" + ] + + kafka-exporter: + image: bitnami/kafka-exporter + container_name: kafkaexporter + command: [ "--kafka.server=127.0.0.1:9092" ] + network_mode: host + expose: + - 9308 + depends_on: + kafka: + condition: service_healthy logprep: build: diff --git a/quickstart/exampledata/config/grafana/dashboards/kafka.json b/quickstart/exampledata/config/grafana/dashboards/kafka.json new file mode 100644 index 000000000..1116843a0 --- /dev/null +++ b/quickstart/exampledata/config/grafana/dashboards/kafka.json @@ -0,0 +1,678 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Kafka resource usage and throughput", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 7589, + "graphTooltip": 0, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "App" + ], + "targetBlank": false, + "title": "Applications", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 14, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 480 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "expr": "sum(rate(kafka_topic_partition_current_offset{instance=\"$instance\", topic=~\"$topic\"}[1m])) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "B" + } + ], + "title": "Message in per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 12, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 480 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "expr": "sum(kafka_consumergroup_lag{instance=\"$instance\",topic=~\"$topic\"}) by (consumergroup, topic) ", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "refId": "A" + } + ], + "title": "Lag by Consumer Group", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 16, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 480 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_topic_partition_current_offset{instance=~'$instance', topic=~\"$topic\"}[1m])) by (topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "range": true, + "refId": "A" + } + ], + "title": "Message in per second", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 18, + "links": [], + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 480 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "editorMode": "code", + "expr": "sum(rate(kafka_consumergroup_current_offset{instance=~'$instance',topic=~\"$topic\"}[1m])) by (consumergroup, topic)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{consumergroup}} (topic: {{topic}})", + "range": true, + "refId": "A" + } + ], + "title": "Message consume per second", + "type": "timeseries" + }, + { + "aliasColors": {}, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 20 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 420, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "10.1.4", + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "expr": "sum by(topic) (kafka_topic_partitions{instance=\"$instance\",topic=~\"$topic\"})", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{topic}}", + "refId": "A" + } + ], + "thresholds": [], + "timeRegions": [], + "title": "Partitions per Topic", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "series", + "show": false, + "values": [ + "current" + ] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "show": true + }, + { + "format": "short", + "logBase": 1, + "show": true + } + ], + "yaxis": { + "align": false + } + } + ], + "refresh": "5s", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Kafka", + "App" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "kafka", + "value": "kafka" + }, + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "definition": "label_values(kafka_topic_partitions,job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "job", + "options": [], + "query": { + "query": "label_values(kafka_topic_partitions,job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "localhost:9308", + "value": "localhost:9308" + }, + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "definition": "label_values(kafka_topic_partitions{job=~\"$job\"},instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "query": "label_values(kafka_topic_partitions{job=~\"$job\"},instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "promz2394" + }, + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Topic", + "multi": true, + "name": "topic", + "options": [], + "query": "label_values(kafka_topic_partition_current_offset{instance='$instance',topic!='__consumer_offsets',topic!='--kafka'}, topic)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kafka Exporter Overview", + "uid": "jwPKIsniz", + "version": 1, + "weekStart": "" +} diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json b/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json index 4d1ecdc84..6e978c600 100644 --- a/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json +++ b/quickstart/exampledata/config/grafana/dashboards/logprep-dashboard.json @@ -18,7 +18,23 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "links": [], + "id": 2, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "App" + ], + "targetBlank": false, + "title": "Applications", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -1561,6 +1577,7 @@ "id": 131, "links": [], "options": { + "alertInstanceLabelFilter": "", "alertName": "", "dashboardAlerts": false, "groupBy": [], @@ -2692,8 +2709,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -2909,8 +2925,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3005,8 +3020,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3101,8 +3115,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3210,8 +3223,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3313,8 +3325,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -3472,8 +3483,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -3554,8 +3564,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "yellow", @@ -3818,8 +3827,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -3930,8 +3938,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4042,8 +4049,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -4100,7 +4106,9 @@ "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": [ + "App" + ], "templating": { "list": [] }, @@ -4112,6 +4120,6 @@ "timezone": "", "title": "Logprep Overview", "uid": "Ijmjihx7z", - "version": 4, + "version": 1, "weekStart": "" } diff --git a/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json b/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json index 5933e9c89..4d74a8db5 100644 --- a/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json +++ b/quickstart/exampledata/config/grafana/dashboards/logprep-processors.json @@ -18,7 +18,23 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "links": [], + "id": 3, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "App" + ], + "targetBlank": false, + "title": "Applications", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], "liveNow": false, "panels": [ { @@ -844,8 +860,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -934,8 +949,7 @@ "mode": "percentage", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "#EAB839", @@ -1075,8 +1089,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1172,8 +1185,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1282,8 +1294,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1390,8 +1401,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1511,8 +1521,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1589,7 +1598,9 @@ "refresh": "5s", "schemaVersion": 38, "style": "dark", - "tags": [], + "tags": [ + "App" + ], "templating": { "list": [ { @@ -1633,6 +1644,6 @@ "timezone": "", "title": "Logprep Processors", "uid": "a6a0934c-8f53-47fb-8b30-2adebed11bd5", - "version": 3, + "version": 1, "weekStart": "" } diff --git a/quickstart/exampledata/config/prometheus/logprep.yml b/quickstart/exampledata/config/prometheus/logprep.yml deleted file mode 100644 index 98a892d9f..000000000 --- a/quickstart/exampledata/config/prometheus/logprep.yml +++ /dev/null @@ -1,3 +0,0 @@ -- targets: - - '127.0.0.1:8000' - labels: [] diff --git a/quickstart/exampledata/config/prometheus/prometheus.yml b/quickstart/exampledata/config/prometheus/prometheus.yml index a381fce33..f3642da75 100644 --- a/quickstart/exampledata/config/prometheus/prometheus.yml +++ b/quickstart/exampledata/config/prometheus/prometheus.yml @@ -30,3 +30,7 @@ scrape_configs: - job_name: "logprep" static_configs: - targets: ["localhost:8000"] + - job_name: "kafka" + metrics_path: "/metrics" + static_configs: + - targets: ["localhost:9308"]