From a20285ee6e9db7917d3eb940faee87050032296e Mon Sep 17 00:00:00 2001 From: whiterabbit1983 Date: Mon, 14 Oct 2024 14:12:56 +0300 Subject: [PATCH] refactor: Refactor garafana and prometheus deployment stuff (#646) > [!IMPORTANT] > Refactor Grafana and Prometheus deployment by consolidating configurations into a single `monitoring` directory and updating related `docker-compose` setups. > > - **Refactoring**: > - Consolidate Grafana and Prometheus configurations into `monitoring` directory. > - Remove `grafana/docker-compose.yml` and `prometheus/docker-compose.yml`. > - Update `docker-compose.yml` to include `monitoring/docker-compose.yml` instead of separate Grafana and Prometheus files. > - **File Moves**: > - Move `grafana/provisioning/datasources/datasource.yml` to `monitoring/grafana/provisioning/datasources/datasource.yml`. > - Move `prometheus/config/prometheus.yml` to `monitoring/prometheus/config/prometheus.yml`. > - **Configuration Changes**: > - Update `monitoring/docker-compose.yml` to include both Grafana and Prometheus services with shared volume configurations. > - Add `depends_on` condition for Prometheus service in `monitoring/docker-compose.yml`. > > This description was created by [Ellipsis](https://www.ellipsis.dev?ref=julep-ai%2Fjulep&utm_source=github&utm_medium=referral) for cb36d21dbaa47c6c4c2c6df757d015637844c844. It will automatically update as commits are pushed. --- docker-compose.yml | 3 +- grafana/docker-compose.yml | 20 -------- monitoring/README.md | 35 +++++--------- monitoring/docker-compose.yml | 46 +++++++++---------- monitoring/grafana/datasource.yml | 9 ---- .../provisioning/datasources/datasource.yml | 0 .../prometheus}/config/prometheus.yml | 0 monitoring/prometheus/prometheus.yml | 21 --------- prometheus/docker-compose.yml | 23 ---------- 9 files changed, 35 insertions(+), 122 deletions(-) delete mode 100644 grafana/docker-compose.yml delete mode 100644 monitoring/grafana/datasource.yml rename {grafana => monitoring/grafana}/provisioning/datasources/datasource.yml (100%) rename {prometheus => monitoring/prometheus}/config/prometheus.yml (100%) delete mode 100644 monitoring/prometheus/prometheus.yml delete mode 100644 prometheus/docker-compose.yml diff --git a/docker-compose.yml b/docker-compose.yml index 350adf626..364eb170c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,7 @@ include: - ./scheduler/docker-compose.yml - ./llm-proxy/docker-compose.yml - ./integrations-service/docker-compose.yml - - ./prometheus/docker-compose.yml - - ./grafana/docker-compose.yml + - ./monitoring/docker-compose.yml - ./blob-store/docker-compose.yml # TODO: Enable after testing diff --git a/grafana/docker-compose.yml b/grafana/docker-compose.yml deleted file mode 100644 index 86723774c..000000000 --- a/grafana/docker-compose.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: grafana - -services: - grafana: - image: grafana/grafana - environment: - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} - - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} - container_name: grafana - ports: - - 3000:3000 - volumes: - - grafana_data:/var/lib/grafana - - ./provisioning:/etc/grafana/provisioning - profiles: - - multi-tenant - -volumes: - grafana_data: - external: true diff --git a/monitoring/README.md b/monitoring/README.md index 9702bf6ff..3366abcd7 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -1,41 +1,28 @@ -# TODO: Fix monitoring services -# SCRUM-25 - # Compose sample ### Prometheus & Grafana Project structure: ``` -. -├── compose.yaml +├── README.md +├── docker-compose.yml ├── grafana -│   └── datasource.yml -├── prometheus -│   └── prometheus.yml -└── README.md +│ └── provisioning +│ └── datasources +│ └── datasource.yml +└── prometheus + └── config + └── prometheus.yml ``` -[_compose.yaml_](compose.yaml) -``` -services: - prometheus: - image: prom/prometheus - ... - ports: - - 9090:9090 - grafana: - image: grafana/grafana - ... - ports: - - 3000:3000 -``` -The compose file defines a stack with two services `prometheus` and `grafana`. +The docker compose file defines a stack with two services `prometheus` and `grafana`. When deploying the stack, docker compose maps port the default ports for each service to the equivalent ports on the host in order to inspect easier the web interface of each service. Make sure the ports 9090 and 3000 on the host are not already in use. ## Deploy with docker compose ``` +$ docker volume create grafana_data +$ docker volume create prometheus_data $ docker compose up -d Creating network "prometheus-grafana_default" with the default driver Creating volume "prometheus-grafana_prom_data" with default driver diff --git a/monitoring/docker-compose.yml b/monitoring/docker-compose.yml index 51cb60f67..26fd6faf0 100644 --- a/monitoring/docker-compose.yml +++ b/monitoring/docker-compose.yml @@ -1,39 +1,39 @@ name: julep-monitoring services: - portainer: - image: portainer/portainer-ce:latest - container_name: portainer - restart: unless-stopped - ports: - - "9000:8000" - - "9443:9443" - volumes: - - portainer_data:/data - - /var/run/docker.sock:/var/run/docker.sock - prometheus: image: prom/prometheus container_name: prometheus - command: - - "--config.file=/etc/prometheus/prometheus.yml" - restart: unless-stopped + profiles: + - multi-tenant + volumes: - - ./prometheus:/etc/prometheus - - prom_data:/prometheus + - ./config/prometheus.yml:/etc/prometheus/prometheus.yml + - prometheus_data:/prometheus + + depends_on: + agents-api-multi-tenant: + condition: service_started + + command: + - '--config.file=/etc/prometheus/prometheus.yml' grafana: image: grafana/grafana + environment: + - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} + - GF_SECURITY_ADMIN_USER=${GRAFANA_ADMIN_USER} container_name: grafana ports: - 3000:3000 - restart: unless-stopped - environment: - - GF_SECURITY_ADMIN_USER=admin - - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD} volumes: - - ./grafana:/etc/grafana/provisioning/datasources + - grafana_data:/var/lib/grafana + - ./provisioning:/etc/grafana/provisioning + profiles: + - multi-tenant volumes: - prom_data: - portainer_data: + prometheus_data: + external: true + grafana_data: + external: true diff --git a/monitoring/grafana/datasource.yml b/monitoring/grafana/datasource.yml deleted file mode 100644 index d7b828686..000000000 --- a/monitoring/grafana/datasource.yml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: 1 - -datasources: -- name: Prometheus - type: prometheus - url: http://prometheus:9090 - isDefault: true - access: proxy - editable: true diff --git a/grafana/provisioning/datasources/datasource.yml b/monitoring/grafana/provisioning/datasources/datasource.yml similarity index 100% rename from grafana/provisioning/datasources/datasource.yml rename to monitoring/grafana/provisioning/datasources/datasource.yml diff --git a/prometheus/config/prometheus.yml b/monitoring/prometheus/config/prometheus.yml similarity index 100% rename from prometheus/config/prometheus.yml rename to monitoring/prometheus/config/prometheus.yml diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml deleted file mode 100644 index dec073bbb..000000000 --- a/monitoring/prometheus/prometheus.yml +++ /dev/null @@ -1,21 +0,0 @@ -global: - scrape_interval: 5s - scrape_timeout: 3s - evaluation_interval: 5s -alerting: - alertmanagers: - - static_configs: - - targets: [] - scheme: http - timeout: 3s - api_version: v1 -scrape_configs: - - job_name: traefik - honor_timestamps: true - scrape_interval: 5s - scrape_timeout: 3s - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - gateway:8082 diff --git a/prometheus/docker-compose.yml b/prometheus/docker-compose.yml deleted file mode 100644 index f3c53d966..000000000 --- a/prometheus/docker-compose.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: prometheus - -services: - prometheus: - image: prom/prometheus - container_name: prometheus - profiles: - - multi-tenant - - volumes: - - ./config/prometheus.yml:/etc/prometheus/prometheus.yml - - prometheus_data:/prometheus - - depends_on: - agents-api-multi-tenant: - condition: service_started - - command: - - '--config.file=/etc/prometheus/prometheus.yml' - -volumes: - prometheus_data: - external: true