From 291b477ff98ab83d5dcd429ea6c2446cda02c3c8 Mon Sep 17 00:00:00 2001 From: David Garvey Date: Mon, 23 Dec 2024 16:50:56 +0100 Subject: [PATCH] implement further examples --- .../healthcheck-blackbox/docker-compose.yml | 3 +- .../dashboards/dashboard-model.json | 12 ++++-- .../provisioning/dashboards/provider.yaml | 2 +- .../provisioning/datasources/prometheus.yaml | 2 +- .../volumes/prometheus/alert.rules | 11 ++++++ .../volumes/prometheus/blackbox.yaml | 14 ++++++- .../volumes/prometheus/prometheus.yaml | 38 +++++++++++++++++-- 7 files changed, 71 insertions(+), 11 deletions(-) create mode 100644 deployments/healthcheck-blackbox/volumes/prometheus/alert.rules diff --git a/deployments/healthcheck-blackbox/docker-compose.yml b/deployments/healthcheck-blackbox/docker-compose.yml index 0bca334a..e04ade46 100755 --- a/deployments/healthcheck-blackbox/docker-compose.yml +++ b/deployments/healthcheck-blackbox/docker-compose.yml @@ -4,6 +4,7 @@ services: image: prom/prometheus:v3.0.1 volumes: - ./deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yml + - ./deployments/healthcheck-blackbox/volumes/prometheus/alert.rules:/etc/prometheus/alert.rules ports: - "9090:9090" command: @@ -36,4 +37,4 @@ services: - tyk volumes: - grafana-data: \ No newline at end of file + grafana-data: diff --git a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/dashboard-model.json b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/dashboard-model.json index 3889a447..628fc7ee 100644 --- a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/dashboard-model.json +++ b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/dashboard-model.json @@ -25,7 +25,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], "panels": [ { @@ -1044,7 +1044,10 @@ "Time 9": true, "Value": true, "Value #A": false, + "Value #B": true, + "Value #C": true, "Value #F": true, + "Value #H": true, "Value #I": true, "Value #J": false, "__name__": true, @@ -1060,6 +1063,7 @@ "__name__ 9": true, "fingerprint_sha256": true, "instance": false, + "issuer": true, "job": true, "job 1": true, "job 10": true, @@ -1082,8 +1086,10 @@ "phase 4": true, "phase 5": true, "subject": true, - "subjectalternative": true + "subjectalternative": true, + "version": true }, + "includeByName": {}, "indexByName": { "Time 1": 15, "Time 10": 43, @@ -2056,6 +2062,6 @@ "timezone": "", "title": "Tyk System Health", "uid": "tyk-system-health", - "version": 1, + "version": 2, "weekStart": "" } \ No newline at end of file diff --git a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/provider.yaml b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/provider.yaml index fe7bf206..10a36ac7 100644 --- a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/provider.yaml +++ b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/provider.yaml @@ -10,4 +10,4 @@ providers: updateIntervalSeconds: 10 allowUiUpdates: true options: - path: /etc/grafana/provisioning/dashboards \ No newline at end of file + path: /etc/grafana/provisioning/dashboards diff --git a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/datasources/prometheus.yaml b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/datasources/prometheus.yaml index 074d649a..dd8da401 100644 --- a/deployments/healthcheck-blackbox/volumes/grafana/provisioning/datasources/prometheus.yaml +++ b/deployments/healthcheck-blackbox/volumes/grafana/provisioning/datasources/prometheus.yaml @@ -9,4 +9,4 @@ datasources: basicAuth: false isDefault: true version: 1 - editable: true \ No newline at end of file + editable: true diff --git a/deployments/healthcheck-blackbox/volumes/prometheus/alert.rules b/deployments/healthcheck-blackbox/volumes/prometheus/alert.rules new file mode 100644 index 00000000..8099a6c7 --- /dev/null +++ b/deployments/healthcheck-blackbox/volumes/prometheus/alert.rules @@ -0,0 +1,11 @@ +groups: + - name: blackbox_alerts + rules: + - alert: BlackboxProbeFailed + expr: probe_success == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Probe failure for {{ $labels.instance }}" + description: "The Blackbox probe for {{ $labels.instance }} has failed for over 2 minutes." diff --git a/deployments/healthcheck-blackbox/volumes/prometheus/blackbox.yaml b/deployments/healthcheck-blackbox/volumes/prometheus/blackbox.yaml index f1cb8b38..1f4e3f21 100644 --- a/deployments/healthcheck-blackbox/volumes/prometheus/blackbox.yaml +++ b/deployments/healthcheck-blackbox/volumes/prometheus/blackbox.yaml @@ -1,5 +1,5 @@ modules: - http_2xx_json: + http_200_json_status_pass: prober: http timeout: 5s http: @@ -8,4 +8,14 @@ modules: valid_status_codes: [200] valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] fail_if_body_not_matches_regexp: - - '{"status":"pass".*}' \ No newline at end of file + - '{"status":"pass".*}' + tls_config: + insecure_skip_verify: true # Disables certificate validation + http_200: + prober: http + timeout: 5s + http: + method: GET + preferred_ip_protocol: ip4 + valid_status_codes: [200] + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] diff --git a/deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml b/deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml index 737e5d28..e1fa1515 100644 --- a/deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml +++ b/deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml @@ -2,14 +2,32 @@ global: scrape_interval: 15s evaluation_interval: 15s +rule_files: + - /etc/prometheus/alert.rules + scrape_configs: - - job_name: 'blackbox' + - job_name: 'gateway_status' metrics_path: /probe params: - module: [http_2xx_json] + module: [http_200_json_status_pass] static_configs: - targets: - http://tyk-gateway:8080/hello + - https://tyk-gateway-2:8080/hello + - http://tyk-worker-gateway:8080/hello + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115 + - job_name: 'mdcb_status' + metrics_path: /probe + params: + module: [http_200_json_status_pass] + static_configs: + - targets: - http://tyk-mdcb:8181/readiness relabel_configs: - source_labels: [__address__] @@ -17,4 +35,18 @@ scrape_configs: - source_labels: [__param_target] target_label: instance - target_label: __address__ - replacement: blackbox-exporter:9115 \ No newline at end of file + replacement: blackbox-exporter:9115 + - job_name: 'api_status' + metrics_path: /probe + params: + module: [http_200] + static_configs: + - targets: + - http://tyk-gateway:8080/basic-open-api/get + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox-exporter:9115