diff --git a/10broker-config.yml b/10broker-config.yml index b86a0577..af0f0374 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -4,6 +4,13 @@ metadata: namespace: kafka apiVersion: v1 data: + init.sh: |- + #!/bin/bash + set -x + + export KAFKA_BROKER_ID=${HOSTNAME##*-} + sed -i "s/\${KAFKA_BROKER_ID}/$KAFKA_BROKER_ID/" /etc/kafka/server.properties + server.properties: |- # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -25,7 +32,7 @@ data: ############################# Server Basics ############################# # The id of the broker. This must be set to a unique integer for each broker. - broker.id=0 + broker.id=${KAFKA_BROKER_ID} # Switch to enable topic deletion or not, default value is false #delete.topic.enable=true diff --git a/50kafka.yml b/50kafka.yml index 8536b1c4..9b8bde28 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -15,6 +15,13 @@ spec: prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 30 + initContainers: + - name: init-config + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: ['/bin/bash', '/etc/kafka/init.sh'] + volumeMounts: + - name: config + mountPath: /etc/kafka containers: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d @@ -31,22 +38,22 @@ spec: image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce env: - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties + value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - name: JMX_PORT value: "5555" ports: - containerPort: 9092 command: - - /bin/sh - - -c - - > - ./bin/kafka-server-start.sh - ./config/server.properties - --override zookeeper.connect=zookeeper:2181 - --override log.retention.hours=-1 - --override log.dirs=/var/lib/kafka/data/topics - --override broker.id=${HOSTNAME##*-} - --override auto.create.topics.enable=false + - ./bin/kafka-server-start.sh + - /etc/kafka/server.properties + - --override + - zookeeper.connect=zookeeper:2181 + - --override + - log.retention.hours=-1 + - --override + - log.dirs=/var/lib/kafka/data/topics + - --override + - auto.create.topics.enable=false resources: requests: cpu: 100m @@ -59,7 +66,7 @@ spec: - 'echo "" | nc -w 1 127.0.0.1 9092' volumeMounts: - name: config - mountPath: /opt/kafka/config + mountPath: /etc/kafka - name: data mountPath: /var/lib/kafka/data volumes: diff --git a/zookeeper/10zookeeper-config.yml b/zookeeper/10zookeeper-config.yml index 58d8b6aa..e796b4ba 100644 --- a/zookeeper/10zookeeper-config.yml +++ b/zookeeper/10zookeeper-config.yml @@ -4,6 +4,15 @@ metadata: namespace: kafka apiVersion: v1 data: + init.sh: |- + #!/bin/bash + set -x + + [ -z "$ID_OFFSET" ] && ID_OFFSET=1 + export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + $ID_OFFSET)) + echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid + sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" /etc/kafka/zookeeper.properties + zookeeper.properties: |- tickTime=2000 dataDir=/var/lib/zookeeper/data @@ -16,9 +25,13 @@ data: server.3=pzoo-2.pzoo:2888:3888:participant server.4=zoo-0.zoo:2888:3888:participant server.5=zoo-1.zoo:2888:3888:participant - + log4j.properties: |- log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c)%n + + # Suppress connection log messages, three lines per livenessProbe execution + log4j.logger.org.apache.zookeeper.server.NIOServerCnxnFactory=WARN + log4j.logger.org.apache.zookeeper.server.NIOServerCnxn=WARN diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index ef12291b..9989662f 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -16,6 +16,15 @@ spec: prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 10 + initContainers: + - name: init-config + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: ['/bin/bash', '/etc/kafka/init.sh'] + volumeMounts: + - name: config + mountPath: /etc/kafka + - name: data + mountPath: /var/lib/zookeeper/data containers: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d @@ -40,18 +49,12 @@ spec: image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce env: - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties + value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - name: JMX_PORT value: "5555" command: - - /bin/sh - - -euc - - > - export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 1)); - echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid; - sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties; - cat config/zookeeper.properties; - ./bin/zookeeper-server-start.sh config/zookeeper.properties + - ./bin/zookeeper-server-start.sh + - /etc/kafka/zookeeper.properties ports: - containerPort: 2181 name: client @@ -77,7 +80,7 @@ spec: - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' volumeMounts: - name: config - mountPath: /opt/kafka/config + mountPath: /etc/kafka - name: data mountPath: /var/lib/zookeeper/data volumes: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index 0f5972cf..7e39cac8 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -16,6 +16,18 @@ spec: prometheus.io/port: "5556" spec: terminationGracePeriodSeconds: 10 + initContainers: + - name: init-config + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: ['/bin/bash', '/etc/kafka/init.sh'] + env: + - name: ID_OFFSET + value: "4" + volumeMounts: + - name: config + mountPath: /etc/kafka + - name: data + mountPath: /var/lib/zookeeper/data containers: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d @@ -40,18 +52,12 @@ spec: image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce env: - name: KAFKA_LOG4J_OPTS - value: -Dlog4j.configuration=file:/opt/kafka/config/log4j.properties + value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties - name: JMX_PORT value: "5555" command: - - /bin/sh - - -euc - - > - export ZOOKEEPER_SERVER_ID=$((${HOSTNAME##*-} + 4)); - echo "${ZOOKEEPER_SERVER_ID:-1}" | tee /var/lib/zookeeper/data/myid; - sed -i "s/server\.$ZOOKEEPER_SERVER_ID\=[a-z0-9.-]*/server.$ZOOKEEPER_SERVER_ID=0.0.0.0/" config/zookeeper.properties; - cat config/zookeeper.properties; - ./bin/zookeeper-server-start.sh config/zookeeper.properties + - ./bin/zookeeper-server-start.sh + - /etc/kafka/zookeeper.properties ports: - containerPort: 2181 name: client @@ -77,7 +83,7 @@ spec: - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' volumeMounts: - name: config - mountPath: /opt/kafka/config + mountPath: /etc/kafka - name: data mountPath: /var/lib/zookeeper/data volumes: