feat: Add prometheus and grafana support

agents-api/agents_api/web.py

@@ -13,6 +13,7 @@
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from litellm.exceptions import APIError
+from prometheus_fastapi_instrumentator import Instrumentator
 from pycozo.client import QueryException
 from scalar_fastapi import get_scalar_api_reference
 from temporalio.service import RPCError
@@ -100,6 +101,9 @@ def register_exceptions(app: FastAPI) -> None:
     root_path=api_prefix,
 )
 
+# Enable metrics
+Instrumentator().instrument(app).expose(app)
+
 # Create a new router for the docs
 scalar_router = APIRouter()
 

agents-api/pyproject.toml

@@ -38,6 +38,7 @@ scalar-fastapi = "^1.0.3"
 sse-starlette = "^2.1.3"
 anyio = "^4.4.0"
 python-box = {extras = ["toml"], version = "^7.2.0"}
+prometheus-fastapi-instrumentator = "^7.0.0"
 [tool.poetry.group.dev.dependencies]
 ipython = "^8.26.0"
 ruff = "^0.5.5"

docker-compose.yml

@@ -12,6 +12,8 @@ include:
   - ./scheduler/docker-compose.yml
   - ./llm-proxy/docker-compose.yml
   - ./integrations-service/docker-compose.yml
+  - ./prometheus/docker-compose.yml
+  - ./grafana/docker-compose.yml
 
   # TODO: Enable after testing
   # - ./monitoring/docker-compose.yml

grafana/docker-compose.yml

@@ -0,0 +1,20 @@
+name: grafana
+
+services:
+  grafana:
+    image: grafana/grafana
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD}
+      - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER}
+    container_name: grafana
+    ports:
+      - 3000:3000
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./provisioning:/etc/grafana/provisioning
+    profiles:
+      - multi-tenant
+
+volumes:
+  grafana_data:
+    external: true

grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,9 @@
+datasources:
+-  access: 'proxy'
+   editable: true
+   is_default: true
+   name: 'agents-api'
+   org_id: 1
+   type: 'prometheus'
+   url: 'http://prometheus:9090'
+   version: 1           

prometheus/config/prometheus.yml

@@ -0,0 +1,24 @@
+global:
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  evaluation_interval: 15s
+alerting:
+  alertmanagers:
+  - follow_redirects: true
+    enable_http2: true
+    scheme: http
+    timeout: 10s
+    api_version: v2
+    static_configs:
+    - targets: []
+scrape_configs:
+- job_name: agents-api
+  honor_timestamps: true
+  scrape_interval: 15s
+  scrape_timeout: 10s
+  metrics_path: /metrics
+  scheme: http
+  follow_redirects: true
+  static_configs:
+  - targets:
+    - agents-api-multi-tenant:8080

prometheus/docker-compose.yml

@@ -0,0 +1,23 @@
+name: prometheus
+
+services:
+  prometheus:
+    image: prom/prometheus
+    container_name: prometheus
+    profiles:
+      - multi-tenant
+
+    volumes:
+      - ./config/prometheus.yml:/etc/prometheus/prometheus.yml
+      - prometheus_data:/prometheus
+
+    depends_on:
+      agents-api-multi-tenant:
+        condition: service_started
+
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+
+volumes:
+  prometheus_data:
+    external: true