forked from nachomdo/draper
-
Notifications
You must be signed in to change notification settings - Fork 5
/
Copy pathdocker-compose.yml
255 lines (240 loc) · 8.66 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
version: '3.8'
services:
##### Applications
api-go-service:
hostname: api-go-service
build:
context: ./goservice
environment:
BOOTSTRAP_SERVERS: broker:29092
OTEL_ENDPOINT: "apm-server:8200"
ports:
- '3100:3100'
depends_on:
ksqldb-server:
condition: service_started
apm-server:
condition: service_healthy
kstream-service:
build:
context: ./kstream-service
container_name: kstream-service
hostname: kstream-service
volumes:
- ./agents:/mnt
environment:
BOOTSTRAP_SERVER: broker:29092
SCHEMA_REGISTRY: http://schema-registry:8081
OTEL_RESOURCE_ATTRIBUTES: "service.name=kstream-service-apm"
OTEL_TRACES_EXPORTER: otlp
OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
OTEL_TRACES_SAMPLER: always_on
JAVA_OPTS: "-javaagent:/mnt/opentelemetry-javaagent-all.jar -javaagent:/mnt/jmx_prometheus_javaagent-0.16.1.jar=9999:/mnt/prometheus/kafka_streams.yml"
depends_on:
schema-registry:
condition: service_healthy
apm-server:
condition: service_healthy
##### Observability Infrastructure
# Elastic Application Performance Monitoring (APM) server.
# OTLP Compliant observability backend.
apm-server:
image: docker.elastic.co/apm/apm-server:7.16.2
container_name: apm-server
hostname: apm-server
depends_on:
elasticsearch:
condition: service_healthy
kibana:
condition: service_healthy
cap_add: ["CHOWN", "DAC_OVERRIDE", "SETGID", "SETUID"]
cap_drop: ["ALL"]
ports:
- 8200:8200
command: >
apm-server -e
-E apm-server.rum.enabled=true
-E setup.kibana.host=kibana:5601
-E setup.template.settings.index.number_of_replicas=0
-E apm-server.kibana.enabled=true
-E apm-server.kibana.host=kibana:5601
-E output.elasticsearch.hosts=["elasticsearch:9200"]
-E output.elasticsearch.bulk_max_size=10000
-E queue.mem.events=10000
healthcheck:
interval: 10s
retries: 20
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:8200/
# Search engine for all Elastic services
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.16.2
container_name: elasticsearch
environment:
- node.name=es01
- bootstrap.memory_lock=true
- cluster.name=docker-cluster
- discovery.type=single-node
- ES_JAVA_OPTS=-Xms1g -Xmx1g
ports:
- 9200:9200
healthcheck:
interval: 10s
retries: 20
test: curl -s http://localhost:9200/_cluster/health | grep -vq '"status":"red"'
# Serves Elastic GUI
kibana:
image: docker.elastic.co/kibana/kibana:7.16.2
container_name: kibana
depends_on:
elasticsearch:
condition: service_healthy
environment:
ELASTICSEARCH_URL: http://elasticsearch:9200
ELASTICSEARCH_HOSTS: http://elasticsearch:9200
ports:
- 5601:5601
healthcheck:
interval: 10s
retries: 20
test: curl --write-out 'HTTP %{http_code}' --fail --silent --output /dev/null http://localhost:5601/api/status
# OpenTelemetry Collector
# Translates JMX metrics from the prometheus exporter agent into OTLP format and sends to apm-server
collector:
image: otel/opentelemetry-collector:latest
container_name: collector
hostname: collector
# Container runs otelcol CLI on launch. Command adds CLI options.
command: ["--config=/etc/collector.yml"]
volumes:
- ./collector.yml:/etc/collector.yml
ports:
- "13133:13133"
- "55680:55680"
- "8889:8889" # used for prometheus metrics debugging
depends_on:
apm-server:
condition: service_healthy
##### Data infrastructure
# Manages broker cluster metadata
zookeeper:
image: confluentinc/cp-zookeeper:6.2.1
hostname: zookeeper
container_name: zookeeper
ports:
- '2181:2181'
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
# Kafka broker -- Stores event streams in topics
broker:
image: confluentinc/cp-server:6.2.1
hostname: broker
container_name: broker
ports:
- '9092:9092'
- '29092:29092'
depends_on:
- zookeeper
environment:
KAFKA_BROKER_ID: 1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
# Runs ksql queries for real-time stream processing
ksqldb-server:
image: confluentinc/cp-ksqldb-server:6.2.1
hostname: ksqldb-server
container_name: ksqldb-server
volumes:
- ./agents:/agents
ports:
- "8088:8088"
environment:
KSQL_BOOTSTRAP_SERVERS: "broker:29092"
KSQL_HOST_NAME: ksqldb-server
KSQL_LISTENERS: "http://0.0.0.0:8088"
KSQL_CACHE_MAX_BYTES_BUFFERING: 0
KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://schema-registry:8081"
KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
KSQL_KSQL_CONNECT_URL: "http://connect:8083"
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'
# # Run script in headless mode
# KSQL_KSQL_QUERIES_FILE: /tmp/ksqldb_script.sql
OTEL_TRACES_EXPORTER: otlp
OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
OTEL_TRACES_SAMPLER: always_on
OTEL_RESOURCE_ATTRIBUTES: "service.name=ksql_db,service.version=1.0,deployment.environment=development"
KSQL_OPTS: "-javaagent:/agents/opentelemetry-javaagent-all.jar"
depends_on:
schema-registry:
condition: service_healthy
# Manages evolving schemas for event streams
schema-registry:
image: confluentinc/cp-schema-registry:6.2.1
hostname: schema-registry
container_name: schema-registry
depends_on:
- broker
volumes:
- ./agents:/tmp
ports:
- "8081:8081"
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'broker:29092'
SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
OTEL_RESOURCE_ATTRIBUTES: "service.name=schema-registry"
OTEL_TRACES_EXPORTER: otlp
OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
OTEL_TRACES_SAMPLER: always_on
SCHEMA_REGISTRY_OPTS: "-javaagent:/tmp/opentelemetry-javaagent-all.jar"
healthcheck:
interval: 10s
retries: 20
test: ["CMD", "curl", "-f", "http://localhost:8081/subjects"]
# Imports/exports data from/to external systems to/from topics
connect:
image: cnfldemos/kafka-connect-datagen:0.5.0-6.2.1
hostname: connect
container_name: connect
depends_on:
schema-registry:
condition: service_healthy
ports:
- "8083:8083"
volumes:
- ./agents:/tmp
environment:
CONNECT_BOOTSTRAP_SERVERS: 'broker:29092'
CONNECT_REST_ADVERTISED_HOST_NAME: connect
CONNECT_REST_PORT: 8083
CONNECT_GROUP_ID: compose-connect-group
CONNECT_CONFIG_STORAGE_TOPIC: docker-connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_OFFSET_STORAGE_TOPIC: docker-connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: docker-connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: io.confluent.connect.avro.AvroConverter
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
CONNECT_LOG4J_LOGGERS: "org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR"
OTEL_RESOURCE_ATTRIBUTES: "service.name=kafka-connect"
OTEL_TRACES_EXPORTER: otlp
OTEL_EXPORTER_OTLP_ENDPOINT: http://apm-server:8200
OTEL_TRACES_SAMPLER: always_on
KAFKA_OPTS: "-javaagent:/tmp/opentelemetry-javaagent-all.jar"
volumes:
ksql_streams_data: