implement further examples

deployments/healthcheck-blackbox/docker-compose.yml

@@ -4,6 +4,7 @@ services:
     image: prom/prometheus:v3.0.1
     volumes:
       - ./deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml:/etc/prometheus/prometheus.yml
+      - ./deployments/healthcheck-blackbox/volumes/prometheus/alert.rules:/etc/prometheus/alert.rules
     ports:
       - "9090:9090"
     command:
@@ -36,4 +37,4 @@ services:
       - tyk 
 
 volumes:
-  grafana-data:
+  grafana-data:

deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/dashboard-model.json

@@ -25,7 +25,7 @@
   "editable": true,
   "fiscalYearStartMonth": 0,
   "graphTooltip": 0,
-  "id": 2,
+  "id": 1,
   "links": [],
   "panels": [
     {
@@ -1044,7 +1044,10 @@
               "Time 9": true,
               "Value": true,
               "Value #A": false,
+              "Value #B": true,
+              "Value #C": true,
               "Value #F": true,
+              "Value #H": true,
               "Value #I": true,
               "Value #J": false,
               "__name__": true,
@@ -1060,6 +1063,7 @@
               "__name__ 9": true,
               "fingerprint_sha256": true,
               "instance": false,
+              "issuer": true,
               "job": true,
               "job 1": true,
               "job 10": true,
@@ -1082,8 +1086,10 @@
               "phase 4": true,
               "phase 5": true,
               "subject": true,
-              "subjectalternative": true
+              "subjectalternative": true,
+              "version": true
             },
+            "includeByName": {},
             "indexByName": {
               "Time 1": 15,
               "Time 10": 43,
@@ -2056,6 +2062,6 @@
   "timezone": "",
   "title": "Tyk System Health",
   "uid": "tyk-system-health",
-  "version": 1,
+  "version": 2,
   "weekStart": ""
 }

deployments/healthcheck-blackbox/volumes/grafana/provisioning/dashboards/provider.yaml

@@ -10,4 +10,4 @@ providers:
     updateIntervalSeconds: 10
     allowUiUpdates: true
     options:
-      path: /etc/grafana/provisioning/dashboards
+      path: /etc/grafana/provisioning/dashboards

deployments/healthcheck-blackbox/volumes/grafana/provisioning/datasources/prometheus.yaml

@@ -9,4 +9,4 @@ datasources:
     basicAuth: false
     isDefault: true
     version: 1
-    editable: true
+    editable: true

deployments/healthcheck-blackbox/volumes/prometheus/alert.rules

@@ -0,0 +1,11 @@
+groups:
+  - name: blackbox_alerts
+    rules:
+      - alert: BlackboxProbeFailed
+        expr: probe_success == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Probe failure for {{ $labels.instance }}"
+          description: "The Blackbox probe for {{ $labels.instance }} has failed for over 2 minutes."

deployments/healthcheck-blackbox/volumes/prometheus/blackbox.yaml

@@ -1,5 +1,5 @@
 modules:
-  http_2xx_json:
+  http_200_json_status_pass:
     prober: http
     timeout: 5s
     http:
@@ -8,4 +8,14 @@ modules:
       valid_status_codes: [200]
       valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
       fail_if_body_not_matches_regexp:
-        - '{"status":"pass".*}'
+        - '{"status":"pass".*}'
+      tls_config:
+        insecure_skip_verify: true  # Disables certificate validation
+  http_200:
+    prober: http
+    timeout: 5s
+    http:
+      method: GET
+      preferred_ip_protocol: ip4
+      valid_status_codes: [200]
+      valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]

deployments/healthcheck-blackbox/volumes/prometheus/prometheus.yaml

@@ -2,19 +2,51 @@ global:
   scrape_interval: 15s
   evaluation_interval: 15s
 
+rule_files:
+  - /etc/prometheus/alert.rules
+
 scrape_configs:
-  - job_name: 'blackbox'
+  - job_name: 'gateway_status'
     metrics_path: /probe
     params:
-      module: [http_2xx_json]
+      module: [http_200_json_status_pass]
     static_configs:
       - targets:
         - http://tyk-gateway:8080/hello
+        - https://tyk-gateway-2:8080/hello
+        - http://tyk-worker-gateway:8080/hello
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115
+  - job_name: 'mdcb_status'
+    metrics_path: /probe
+    params:
+      module: [http_200_json_status_pass]
+    static_configs:
+      - targets:
         - http://tyk-mdcb:8181/readiness
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target
       - source_labels: [__param_target]
         target_label: instance
       - target_label: __address__
-        replacement: blackbox-exporter:9115
+        replacement: blackbox-exporter:9115
+  - job_name: 'api_status'
+    metrics_path: /probe
+    params:
+      module: [http_200]
+    static_configs:
+      - targets:
+        - http://tyk-gateway:8080/basic-open-api/get
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox-exporter:9115