From 2f772c26e1949ba6258689d77b28beb632bebd79 Mon Sep 17 00:00:00 2001 From: akchainsafe Date: Thu, 7 Nov 2024 14:56:08 +0700 Subject: [PATCH] chore: added loki logs to example docker-compose (#385) --- example/docker-compose.yml | 104 ++++++++++++++++++++--- example/grafana/datasources/grafana.yaml | 10 +++ example/grafana/plugins/app.yaml | 12 +++ example/loki/loki.yaml | 49 +++++++++++ example/promtail/promtail.yaml | 39 +++++++++ 5 files changed, 203 insertions(+), 11 deletions(-) create mode 100644 example/grafana/datasources/grafana.yaml create mode 100644 example/grafana/plugins/app.yaml create mode 100644 example/loki/loki.yaml create mode 100644 example/promtail/promtail.yaml diff --git a/example/docker-compose.yml b/example/docker-compose.yml index 2f228bc4..3a2dda73 100644 --- a/example/docker-compose.yml +++ b/example/docker-compose.yml @@ -1,20 +1,36 @@ # Copyright 2020 ChainSafe Systems # SPDX-License-Identifier: LGPL-3.0-only +x-logging: + &default-logging + driver: "json-file" + options: + max-size: "1m" + max-file: "1" + tag: "{{.Name}}" + services: mempool-stub: image: ghcr.io/sygmaprotocol/beacon-api-stub + container_name: mempool-stub + labels: + logging: "promtail" + logging_jobname: "containerlogs" volumes: - "./cfg/stubs:/stubs" ports: - 8882:8882 environment: - STUB_DATA=/stubs/mempool.yml + logging: *default-logging bitcoin: # image: ruimarinho/bitcoin-core:latest image: ghcr.io/sygmaprotocol/bitcoin container_name: bitcoin + labels: + logging: "promtail" + logging_jobname: "containerlogs" ports: - "18443:18443" - "18332:18332" @@ -27,13 +43,16 @@ services: volumes: - ./cfg/entrypoint/entrypoint.sh:/cfg/entrypoint/entrypoint.sh entrypoint: /cfg/entrypoint/entrypoint.sh + logging: *default-logging evm1-1: image: ghcr.io/sygmaprotocol/sygma-solidity:evm1-v2.10.1 container_name: evm1-1 command: --chain.chainId 1337 --db data/ --blockTime 2 --m 'black toward wish jar twin produce remember fluid always confirm bacon slush' > /dev/null - logging: - driver: none + labels: + logging: "promtail" + logging_jobname: "containerlogs" + logging: *default-logging ports: - "8545:8545" @@ -41,26 +60,35 @@ services: image: ghcr.io/sygmaprotocol/sygma-solidity:evm2-v2.10.1 command: --chain.chainId 1338 --db data/ --blockTime 2 --m 'black toward wish jar twin produce remember fluid always confirm bacon slush' > /dev/null container_name: evm2-1 - logging: - driver: none + labels: + logging: "promtail" + logging_jobname: "containerlogs" + logging: *default-logging ports: - "8547:8545" substrate-pallet: - container_name: substrate-pallet image: "ghcr.io/sygmaprotocol/sygma-substrate-pallets:e2e-v0.3.1" + container_name: substrate-pallet + labels: + logging: "promtail" + logging_jobname: "containerlogs" ports: - "9944:9944" - "9933:9933" - + logging: *default-logging + ipfs0: - container_name: ipfs0 image: ipfs/kubo:release + container_name: ipfs0 + labels: + logging: "promtail" + logging_jobname: "containerlogs" ports: - - "4001:4001" - - "5002:5001" - - "8080:8080" - + - "4001:4001" + - "5002:5001" + - "8080:8080" + logging: *default-logging relayer1: container_name: example_relayer1 @@ -71,9 +99,13 @@ services: depends_on: - evm1-1 - substrate-pallet + labels: + logging: "promtail" + logging_jobname: "containerlogs" volumes: - ./cfg:/cfg restart: always + logging: *default-logging relayer2: container_name: example_relayer2 @@ -84,9 +116,13 @@ services: depends_on: - evm1-1 - substrate-pallet + labels: + logging: "promtail" + logging_jobname: "containerlogs" volumes: - ./cfg:/cfg restart: always + logging: *default-logging relayer3: container_name: example_relayer3 @@ -97,15 +133,61 @@ services: depends_on: - evm1-1 - substrate-pallet + labels: + logging: "promtail" + logging_jobname: "containerlogs" volumes: - ./cfg:/cfg restart: always + logging: *default-logging otel-collector: container_name: otel-collector image: otel/opentelemetry-collector command: ["--config=/etc/otel-collector-config.yml"] + labels: + logging: "promtail" + logging_jobname: "containerlogs" volumes: - ./cfg/otel-collector-config.yml:/etc/otel-collector-config.yml ports: - "8889:8889" # Prometheus exporter metrics + logging: *default-logging + + grafana: + image: grafana/grafana:11.3.0 + container_name: grafana + ports: + - 3000:3000 + volumes: + - ./grafana/datasources/grafana.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + - ./grafana/plugins/app.yaml:/etc/grafana/provisioning/plugins/app.yaml + environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_DISABLE_LOGIN_FORM=true + - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane + - GF_INSTALL_PLUGINS=grafana-lokiexplore-app + logging: *default-logging + + loki: + image: grafana/loki:3.0.0 + container_name: loki + ports: + - 3100:3100 + command: -config.file=/etc/loki/loki.yaml + volumes: + - ./loki/loki.yaml:/etc/loki/loki.yaml + logging: *default-logging + + promtail: + image: grafana/promtail:3.0.0 + container_name: promtail + volumes: + - ./promtail/promtail.yaml:/etc/promtail/docker-config.yaml + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + command: -config.file=/etc/promtail/docker-config.yaml + depends_on: + - loki + logging: *default-logging \ No newline at end of file diff --git a/example/grafana/datasources/grafana.yaml b/example/grafana/datasources/grafana.yaml new file mode 100644 index 00000000..868c26f1 --- /dev/null +++ b/example/grafana/datasources/grafana.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +datasources: + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + version: 1 + editable: false + isDefault: true \ No newline at end of file diff --git a/example/grafana/plugins/app.yaml b/example/grafana/plugins/app.yaml new file mode 100644 index 00000000..64f8efd1 --- /dev/null +++ b/example/grafana/plugins/app.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +apps: + - type: "grafana-lokiexplore-app" + org_id: 1 + org_name: "Grafana" + disabled: false + jsonData: + apiUrl: http://localhost + isApiKeySet: true + secureJsonData: + apiKey: example-key diff --git a/example/loki/loki.yaml b/example/loki/loki.yaml new file mode 100644 index 00000000..1cd1f51c --- /dev/null +++ b/example/loki/loki.yaml @@ -0,0 +1,49 @@ +auth_enabled: false + +server: + http_listen_port: 3100 + grpc_listen_port: 9096 + +common: + instance_addr: 127.0.0.1 + path_prefix: /tmp/loki + storage: + filesystem: + chunks_directory: /tmp/loki/chunks + rules_directory: /tmp/loki/rules + replication_factor: 1 + ring: + kvstore: + store: inmemory + +frontend: + max_outstanding_per_tenant: 2048 + +pattern_ingester: + enabled: true + +limits_config: + max_global_streams_per_user: 0 + ingestion_rate_mb: 50000 + ingestion_burst_size_mb: 50000 + volume_enabled: true + +query_range: + results_cache: + cache: + embedded_cache: + enabled: true + max_size_mb: 100 + +schema_config: + configs: + - from: 2020-10-24 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +analytics: + reporting_enabled: false diff --git a/example/promtail/promtail.yaml b/example/promtail/promtail.yaml new file mode 100644 index 00000000..b1f97593 --- /dev/null +++ b/example/promtail/promtail.yaml @@ -0,0 +1,39 @@ +# https://grafana.com/docs/loki/latest/clients/promtail/configuration/ +# https://docs.docker.com/engine/api/v1.41/#operation/ContainerList +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: flog_scrape + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_log_stream'] + target_label: 'logstream' + - source_labels: ['__meta_docker_container_label_logging_jobname'] + target_label: 'job' + pipeline_stages: + - cri: {} + - multiline: + firstline: ^\d{4}-\d{2}-\d{2} \d{1,2}:\d{2}:\d{2},\d{3} + max_wait_time: 3s + # https://grafana.com/docs/loki/latest/clients/promtail/stages/json/ + - json: + expressions: + #message: message + level: level + #output: 'message'