-
Notifications
You must be signed in to change notification settings - Fork 3
/
Copy pathdocker-compose.yml
210 lines (191 loc) · 8.08 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
---
version: '3.4'
services:
# Data Stores
## Kafka
zk1:
image: confluentinc/cp-zookeeper:${CP_VERSION}
ports:
- 2181:2181
environment:
ZOOKEEPER_CLIENT_PORT: 2181
ZOOKEEPER_TICK_TIME: 2000
ZOOKEEPER_AUTH_PROVIDER_SASL: org.apache.zookeeper.server.auth.SASLAuthenticationProvider
ZOOKEEPER_REQUIRE_CLIENT_AUTH_SCHEME: sasl
EXTRA_ARGS: "
-Djava.security.auth.login.config=/etc/kafka/zookeeper_jaas.conf
-javaagent:/opt/prometheus-jmx-exporter/jmx_prometheus_javaagent.jar=1234:/opt/prometheus-jmx-exporter/zookeeper.yml
"
volumes:
- ./prometheus-jmx-exporter:/opt/prometheus-jmx-exporter
- ./zk/zookeeper_jaas.conf:/etc/kafka/zookeeper_jaas.conf
kafka1:
image: confluentinc/cp-kafka:${CP_VERSION}
ports:
- 9092:9092
- 9093:9093
environment:
KAFKA_BROKER_ID: 1
KAFKA_NUM_PARTITIONS: 12
KAFKA_ZOOKEEPER_CONNECT: zk1:2181
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_PLAINTEXT_HOST:SASL_PLAINTEXT
KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:19092,PLAINTEXT_HOST://localhost:9092,SASL_PLAINTEXT://kafka1:19093,SASL_PLAINTEXT_HOST://localhost:9093
KAFKA_SASL_ENABLED_MECHANISMS: PLAIN
KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
KAFKA_OPTS: "
-Djava.security.auth.login.config=/etc/kafka/kafka_jaas.conf
-javaagent:/opt/prometheus-jmx-exporter/jmx_prometheus_javaagent.jar=1234:/opt/prometheus-jmx-exporter/kafka-broker.yaml
# "
volumes:
- ./prometheus-jmx-exporter:/opt/prometheus-jmx-exporter
- ./sasl-plain/kafka_jaas.conf:/etc/kafka/kafka_jaas.conf
schema-registry:
image: confluentinc/cp-schema-registry:${CP_VERSION}
ports:
- 8081:8081
environment:
SCHEMA_REGISTRY_HOST_NAME: schema-registry
SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka1:19092
# Connectors
## Kafka Connect cluster Pointed to Confluent Cloud
## Kafka Connect cluster
connect1:
image: confluentinc/cp-kafka-connect:${CP_VERSION}
container_name: connect1
ports:
- 8083:8083
- 5083:5083
environment:
CONNECT_BOOTSTRAP_SERVERS: $CCLOUD_BOOTSTRAP_SERVERS
CONNECT_GROUP_ID: "connect"
CONNECT_CONFIG_STORAGE_TOPIC: demo-connect-configs
CONNECT_OFFSET_STORAGE_TOPIC: demo-connect-offsets
CONNECT_STATUS_STORAGE_TOPIC: demo-connect-status
CONNECT_REPLICATION_FACTOR: 3
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 3
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 3
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 3
CONNECT_OFFSET_STORAGE_PARTITIONS: 3
CONNECT_SECURITY_PROTOCOL: SASL_SSL
CONNECT_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$CCLOUD_API_KEY\" password=\"$CCLOUD_API_SECRET\";"
CONNECT_SASL_MECHANISM: PLAIN
CONNECT_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
CONNECT_REST_ADVERTISED_HOST_NAME: localhost
CONNECT_REST_PORT: 8083
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "false"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: $CCLOUD_SR_URL
CONNECT_VALUE_CONVERTER_BASIC_AUTH_CREDENTIALS_SOURCE: USER_INFO
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_BASIC_AUTH_USER_INFO: "$CCLOUD_SR_API_KEY:$CCLOUD_SR_API_SECRET"
# Connect producer
CONNECT_PRODUCER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
CONNECT_PRODUCER_SECURITY_PROTOCOL: SASL_SSL
CONNECT_PRODUCER_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$CCLOUD_API_KEY\" password=\"$CCLOUD_API_SECRET\";"
CONNECT_PRODUCER_SASL_MECHANISM: PLAIN
CONNECT_PRODUCER_REQUEST_TIMEOUT_MS: "20000"
CONNECT_PRODUCER_RETRY_BACKOFF_MS: "500"
# Connect consumer
CONNECT_CONSUMER_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: "HTTPS"
CONNECT_CONSUMER_SECURITY_PROTOCOL: SASL_SSL
CONNECT_CONSUMER_SASL_JAAS_CONFIG: "org.apache.kafka.common.security.plain.PlainLoginModule required username=\"$CCLOUD_API_KEY\" password=\"$CCLOUD_API_SECRET\";"
CONNECT_CONSUMER_SASL_MECHANISM: PLAIN
CONNECT_CONSUMER_REQUEST_TIMEOUT_MS: "20000"
CONNECT_CONSUMER_RETRY_BACKOFF_MS: "500"
KAFKA_CONNECT_CUB_KAFKA_TIMEOUT: "1"
# connect logs
CONNECT_PLUGIN_PATH: /usr/share/java,/connector-plugins
KAFKA_OPTS: "-javaagent:/opt/prometheus-jmx-exporter/jmx_prometheus_javaagent.jar=5083:/opt/prometheus-jmx-exporter/kafka-connect-v2.yml"
KAFKA_JMX_OPTS: "-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.host=127.0.0.1 -Dcom.sun.management.jmxremote.port=5084 -Dcom.sun.management.jmxremote.rmi.port=5084 -Dcom.sun.management.jmxremote.local.only=false"
volumes:
- ./prometheus-jmx-exporter:/opt/prometheus-jmx-exporter
- ./connector-plugins:/connector-plugins
## Kafka Connect cluster Pointed to Confluent Platform
connect2:
image: confluentinc/cp-kafka-connect:${CP_VERSION}
ports:
- 8084:8084
- 5084:5084
environment:
CONNECT_BOOTSTRAP_SERVERS: kafka1:19092
CONNECT_REST_ADVERTISED_HOST_NAME: localhost
CONNECT_REST_PORT: 8084
CONNECT_GROUP_ID: connect
CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
CONNECT_STATUS_STORAGE_TOPIC: connect-status
CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
CONNECT_KEY_CONVERTER: org.apache.kafka.connect.storage.StringConverter
CONNECT_VALUE_CONVERTER: "io.confluent.connect.avro.AvroConverter"
CONNECT_VALUE_CONVERTER_SCHEMAS_ENABLE: "true"
CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://schema-registry:8081
CONNECT_PLUGIN_PATH: /usr/share/java,/connector-plugins
KAFKA_OPTS: "-javaagent:/opt/prometheus-jmx-exporter/jmx_prometheus_javaagent.jar=5084:/opt/prometheus-jmx-exporter/kafka-connect.yaml"
volumes:
- ./prometheus-jmx-exporter:/opt/prometheus-jmx-exporter
- ./connector-plugins:/connector-plugins
# CICD
julie-ops:
image: purbon/kafka-topology-builder:latest
container_name: julie-ops
command: /bin/bash
stdin_open: true
tty: true
volumes:
- ./topologies:/topologies
# Local database
mysql:
image: mysql
hostname: mysql
container_name: mysql
environment:
MYSQL_ROOT_PASSWORD: changeme
MYSQL_DATABASE: db
MYSQL_USER: user
MYSQL_PASSWORD: changeme
MYSQL_ROOT_HOST: 172.% # Allow docker containers to connect to mysql
ports:
- 3306:3306
volumes:
- ./mysql/init.sql:/docker-entrypoint-initdb.d/init.sql #persistent mysql db data
# Monitoring
grafana:
image: grafana/grafana
container_name: grafana
environment:
GF_INSTALL_PLUGINS: grafana-piechart-panel
ports:
- 3000:3000
volumes:
- ./grafana:/etc/grafana/provisioning/
prometheus:
image: prom/prometheus:latest
container_name: prometheus
ports:
- 9090:9090
volumes:
- ./prometheus-jmx-exporter:/etc/prometheus
mailhog:
image: mailhog/mailhog
container_name: mailhog
logging:
driver: 'none' # disable saving logs
ports:
- 1025:1025 # smtp server
- 8025:8025 # web ui
alertmanager:
image: prom/alertmanager:v0.21.0
container_name: alertmanager
ports:
- 9095:9095
volumes:
- ./alertmanager:/etc/alertmanager