From f5fc72c17115cba9482b440221717ce134c62914 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 05:48:18 +0200
Subject: [PATCH 01/28] Runs the official Kafka distribution on the new Debian
 stable

---
 kafka/Dockerfile | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 4d43b54..3cfc3ab 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,13 +1,19 @@
-FROM openjdk:8-jre-alpine
+FROM debian:stable-20170620
 
-ARG kafka_version=0.10.2.0
-ENV kafka_bin_version=2.12-$kafka_version
+ENV KAFKA_VERSION=0.11.0.0
+ENV kafka_bin_version=2.12-$KAFKA_VERSION
 
-RUN apk add --no-cache --update-cache --virtual build-dependencies curl ca-certificates \
-  && mkdir -p /opt/kafka \
-  && curl -SLs "https://www-eu.apache.org/dist/kafka/$kafka_version/kafka_$kafka_bin_version.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka \
-  && apk del build-dependencies \
-  && rm -rf /var/cache/apk/*
+RUN set -ex; \
+  runDeps='default-jre'; \
+  buildDeps='curl ca-certificates'; \
+  apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
+  rm -rf /var/lib/apt/lists/*; \
+  \
+  mkdir -p /opt/kafka; \
+  curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$kafka_bin_version.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
+  \
+  apt-get purge -y --auto-remove $buildDeps; \
+  rm /var/log/dpkg.log /var/log/apt/*.log
 
 WORKDIR /opt/kafka
 ENTRYPOINT ["bin/kafka-server-start.sh"]

From eeee33b3e3587cc6f6217b594afe9b244e37a9a1 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 05:53:11 +0200
Subject: [PATCH 02/28] https://kafka.apache.org/downloads says "2.11 is
 recommended"

---
 kafka/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 3cfc3ab..a0f08df 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,7 +1,7 @@
 FROM debian:stable-20170620
 
 ENV KAFKA_VERSION=0.11.0.0
-ENV kafka_bin_version=2.12-$KAFKA_VERSION
+ENV kafka_bin_version=2.11-$KAFKA_VERSION
 
 RUN set -ex; \
   runDeps='default-jre'; \

From 99f4d45a138c112c54941191bacff4220f900194 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 06:41:09 +0200
Subject: [PATCH 03/28] Uses Zulu jre/jdk like cp-docker-images base does

---
 kafka/Dockerfile | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index a0f08df..0ec0c2f 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,18 +1,22 @@
 FROM debian:stable-20170620
 
-ENV KAFKA_VERSION=0.11.0.0
+ENV KAFKA_VERSION=0.11.0.0 ZULU_OPENJDK_VERSION="8=8.21.0.1"
 ENV kafka_bin_version=2.11-$KAFKA_VERSION
 
 RUN set -ex; \
-  runDeps='default-jre'; \
-  buildDeps='curl ca-certificates'; \
+  runDeps=''; \
+  buildDeps='curl ca-certificates gnupg dirmngr'; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
-  rm -rf /var/lib/apt/lists/*; \
+  \
+  apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9; \
+  echo 'deb http://repos.azulsystems.com/debian stable main' > /etc/apt/sources.list.d/zulu.list; \
+  apt-get update && apt-get install -y zulu-${ZULU_OPENJDK_VERSION} --no-install-recommends; \
   \
   mkdir -p /opt/kafka; \
   curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$kafka_bin_version.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \
   apt-get purge -y --auto-remove $buildDeps; \
+  rm -rf /var/lib/apt/lists/*; \
   rm /var/log/dpkg.log /var/log/apt/*.log
 
 WORKDIR /opt/kafka

From 8bf55a15576a617f9f5513153c055a78e31b9164 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 06:59:17 +0200
Subject: [PATCH 04/28] Drops a layer

---
 kafka/Dockerfile | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 0ec0c2f..ba412e4 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,7 +1,6 @@
 FROM debian:stable-20170620
 
-ENV KAFKA_VERSION=0.11.0.0 ZULU_OPENJDK_VERSION="8=8.21.0.1"
-ENV kafka_bin_version=2.11-$KAFKA_VERSION
+ENV KAFKA_VERSION=0.11.0.0 ZULU_OPENJDK_VERSION="8=8.21.0.1" KAFKA_BIN_SCALA_VERSION=2.11
 
 RUN set -ex; \
   runDeps=''; \
@@ -13,7 +12,7 @@ RUN set -ex; \
   apt-get update && apt-get install -y zulu-${ZULU_OPENJDK_VERSION} --no-install-recommends; \
   \
   mkdir -p /opt/kafka; \
-  curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$kafka_bin_version.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
+  curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$KAFKA_BIN_SCALA_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \
   apt-get purge -y --auto-remove $buildDeps; \
   rm -rf /var/lib/apt/lists/*; \

From c4066a75693f849e2efcf7397ca75213c69b166e Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 10:24:23 +0200
Subject: [PATCH 05/28] Starts a build-contract to smoke test and exemplify
 image use cases

---
 README.md                          | 12 +++++++++-
 build-contracts/docker-compose.yml | 38 ++++++++++++++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 build-contracts/docker-compose.yml

diff --git a/README.md b/README.md
index d77cfc4..ebad24b 100644
--- a/README.md
+++ b/README.md
@@ -8,4 +8,14 @@ see https://github.com/solsson/dockerfiles/tree/misc-dockerfiles.
 # Kafka docker builds
 
 This repository maintains automated [Kafka](http://kafka.apache.org/) builds for https://hub.docker.com/r/solsson/kafka/
-and related `kafka-` images under https://hub.docker.com/u/solsson/, used with https://github.com/Yolean/kubernetes-kafka/.
+and related `kafka-` images under https://hub.docker.com/u/solsson/.
+
+These images are tested in production with https://github.com/Yolean/kubernetes-kafka/.
+
+## Building
+
+Rudimentary compliance with kubernetes-kafka is tested using a [build-contract](https://github.com/Yolean/build-contract/).
+
+Build and test using: `docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source solsson/build-contract test`
+
+To keep kafka running for local use, uncomment `ports` 9092 and run: `docker-compose -f build-contracts/docker-compose.yml`.
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
new file mode 100644
index 0000000..8ef2b65
--- /dev/null
+++ b/build-contracts/docker-compose.yml
@@ -0,0 +1,38 @@
+version: '2.0'
+services:
+
+  zookeeper:
+    build: ../kafka
+    entrypoint: ./bin/zookeeper-server-start.sh
+    command:
+      - config/zookeeper.properties
+
+  kafka-1:
+    build: ../kafka
+    command:
+      - config/server.properties
+      - --override
+      -   broker.id=1
+      # unlike Kubernetes StatefulSet, compose gives containers a reandom hostname (leading to redirects to a hex name)
+      - --override
+      -   advertised.listener=PLAINTEXT://kafka-1:9092
+
+  test-topic-create-1:
+    build: ../kafka
+    labels:
+      com.yolean.build-contract: ""
+    entrypoint: ./bin/kafka-topics.sh
+    command:
+      - --zookeeper
+      -   zookeeper:2181
+      - --create
+      - --topic test1
+
+  test-snappy-compression:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    entrypoint: /bin/sh
+    command:
+      - -exc
+      - sleep 5; echo "Message from $${HOSTNAME} at $$(date)" | kafkacat -z snappy -b kafka-1:9092 -t test1 -P

From 028aed1c184dc537f781a25f25e81953e24c3b21 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 13:07:58 +0200
Subject: [PATCH 06/28] Fixes topic creation, but with startup timing issues
 because the script doesn't retry

---
 README.md                          |  9 ++++++++-
 build-contracts/docker-compose.yml | 21 +++++++++++++++------
 2 files changed, 23 insertions(+), 7 deletions(-)

diff --git a/README.md b/README.md
index ebad24b..717622a 100644
--- a/README.md
+++ b/README.md
@@ -18,4 +18,11 @@ Rudimentary compliance with kubernetes-kafka is tested using a [build-contract](
 
 Build and test using: `docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source solsson/build-contract test`
 
-To keep kafka running for local use, uncomment `ports` 9092 and run: `docker-compose -f build-contracts/docker-compose.yml`.
+To keep kafka running for local use, uncomment `ports` 9092 and run: `docker-compose -f build-contracts/docker-compose.yml up --force-recreate`.
+
+While timing issues remain, start services individually
+```bash
+compose='docker-compose -f build-contracts/docker-compose.yml'
+$compose up -d zookeeper kafka-0
+$compose up test-topic-create-1
+```
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index 8ef2b65..eaf197f 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -7,26 +7,35 @@ services:
     command:
       - config/zookeeper.properties
 
-  kafka-1:
+  kafka-0:
     build: ../kafka
     command:
       - config/server.properties
       - --override
-      -   broker.id=1
-      # unlike Kubernetes StatefulSet, compose gives containers a reandom hostname (leading to redirects to a hex name)
+      -   broker.id=0
+      # unlike Kubernetes StatefulSet, compose gives containers a random hostname (leading to redirects to a hex name)
       - --override
-      -   advertised.listener=PLAINTEXT://kafka-1:9092
+      -   advertised.listener=PLAINTEXT://kafka-0:9092
+      - --override
+      -   auto.create.topics.enable=false
 
   test-topic-create-1:
     build: ../kafka
     labels:
       com.yolean.build-contract: ""
+    links:
+      - kafka-0
     entrypoint: ./bin/kafka-topics.sh
     command:
       - --zookeeper
       -   zookeeper:2181
       - --create
-      - --topic test1
+      - --topic
+      -   test1
+      - --partitions
+      -   '1'
+      - --replication-factor
+      -   '1'
 
   test-snappy-compression:
     image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
@@ -35,4 +44,4 @@ services:
     entrypoint: /bin/sh
     command:
       - -exc
-      - sleep 5; echo "Message from $${HOSTNAME} at $$(date)" | kafkacat -z snappy -b kafka-1:9092 -t test1 -P
+      - sleep 5; echo "Message from $${HOSTNAME} at $$(date)" | kafkacat -z snappy -b kafka-0:9092 -t test1 -P

From f52bd3b4aada6fc0b1718a6fe51ee7a561439da8 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 13:16:57 +0200
Subject: [PATCH 07/28] Adds a consumer

---
 README.md                          |  1 +
 build-contracts/docker-compose.yml | 14 ++++++++++++++
 2 files changed, 15 insertions(+)

diff --git a/README.md b/README.md
index 717622a..5cb548f 100644
--- a/README.md
+++ b/README.md
@@ -25,4 +25,5 @@ While timing issues remain, start services individually
 compose='docker-compose -f build-contracts/docker-compose.yml'
 $compose up -d zookeeper kafka-0
 $compose up test-topic-create-1
+$compose up test-snappy-compression
 ```
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index eaf197f..d7187ac 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -45,3 +45,17 @@ services:
     command:
       - -exc
       - sleep 5; echo "Message from $${HOSTNAME} at $$(date)" | kafkacat -z snappy -b kafka-0:9092 -t test1 -P
+
+  test-consume-all:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - -b
+      - kafka-0:9092
+      - -t
+      - test1
+      - -C
+      - -o
+      - beginning
+      - -e

From bdea1ad1375506adea16f29d8f8ed66c14483409 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 14:02:37 +0200
Subject: [PATCH 08/28] For previous commit

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 5cb548f..5df2e64 100644
--- a/README.md
+++ b/README.md
@@ -26,4 +26,5 @@ compose='docker-compose -f build-contracts/docker-compose.yml'
 $compose up -d zookeeper kafka-0
 $compose up test-topic-create-1
 $compose up test-snappy-compression
+$compose up test-consume-all
 ```

From ea54416aa50db811387fce6f248dbcff459212c0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 14:03:06 +0200
Subject: [PATCH 09/28] Quick attempt to verify that the image works with
 Connect. Class doesn't load.

---
 kafka-connect-jmx/Dockerfile | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 kafka-connect-jmx/Dockerfile

diff --git a/kafka-connect-jmx/Dockerfile b/kafka-connect-jmx/Dockerfile
new file mode 100644
index 0000000..771ca5d
--- /dev/null
+++ b/kafka-connect-jmx/Dockerfile
@@ -0,0 +1,19 @@
+FROM solsson/kafka
+
+RUN apt-get update && apt-get install -y maven --no-install-recommends
+
+# TODO sort out where Zulu is, and/or tools.jar
+RUN apt-get update && apt-get install -y openjdk-8-jdk-headless --no-install-recommends
+
+RUN set -ex; \
+  mkdir /usr/src/kafka-connect; \
+  cd /usr/src/kafka-connect; \
+  curl -SLs https://github.com/srijiths/kafka-connectors/archive/dc0a7122650e697d3ae97c970a4785bbed949479.tar.gz \
+    | tar -xzf - --strip-components=1 -C ./; \
+  mvn clean install;
+
+# /usr/share/java is the recommendation from http://docs.confluent.io/current/cp-docker-images/docs/intro.html#important-notes-caveats
+RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar /usr/share/java/
+
+ENTRYPOINT ["./bin/connect-standalone.sh"]
+CMD ["./config/connect-standalone.properties", "/usr/src/kafka-connect/kafka-connect-jmx/config/sample-jmx-configuration.properties"]

From 16ef01bb3afc4675a78f1cc2fdb5d9d29014e38c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 14:33:05 +0200
Subject: [PATCH 10/28] Allow automatic topic creation again, to simplify test
 setup

---
 build-contracts/docker-compose.yml | 21 +++++++++++++++++----
 kafka-connect-jmx/Dockerfile       |  7 +++----
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index d7187ac..0247a09 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -16,10 +16,8 @@ services:
       # unlike Kubernetes StatefulSet, compose gives containers a random hostname (leading to redirects to a hex name)
       - --override
       -   advertised.listener=PLAINTEXT://kafka-0:9092
-      - --override
-      -   auto.create.topics.enable=false
 
-  test-topic-create-1:
+  test-topic-create:
     build: ../kafka
     labels:
       com.yolean.build-contract: ""
@@ -31,7 +29,7 @@ services:
       -   zookeeper:2181
       - --create
       - --topic
-      -   test1
+      -   test-topic-create
       - --partitions
       -   '1'
       - --replication-factor
@@ -59,3 +57,18 @@ services:
       - -o
       - beginning
       - -e
+
+  test-kafka-connect:
+    build: ../kafka
+    labels:
+      com.yolean.build-contract: ""
+    links:
+      - kafka-0
+    entrypoint: ./bin/connect-standalone.sh
+    command:
+      # TODO we need to configure bootstrap.servers=kafka-0:9092 here
+      - ./config/connect-standalone.properties
+      - ./config/connect-file-source.properties
+
+  test-kafka-connect-build:
+    build: ../kafka-connect-jmx
diff --git a/kafka-connect-jmx/Dockerfile b/kafka-connect-jmx/Dockerfile
index 771ca5d..9c5f23c 100644
--- a/kafka-connect-jmx/Dockerfile
+++ b/kafka-connect-jmx/Dockerfile
@@ -1,6 +1,6 @@
-FROM solsson/kafka
+FROM solsson/kafka:0.11.0.0
 
-RUN apt-get update && apt-get install -y maven --no-install-recommends
+RUN apt-get update && apt-get install -y maven curl ca-certificates --no-install-recommends
 
 # TODO sort out where Zulu is, and/or tools.jar
 RUN apt-get update && apt-get install -y openjdk-8-jdk-headless --no-install-recommends
@@ -12,8 +12,7 @@ RUN set -ex; \
     | tar -xzf - --strip-components=1 -C ./; \
   mvn clean install;
 
-# /usr/share/java is the recommendation from http://docs.confluent.io/current/cp-docker-images/docs/intro.html#important-notes-caveats
-RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar /usr/share/java/
+RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar ./libs/
 
 ENTRYPOINT ["./bin/connect-standalone.sh"]
 CMD ["./config/connect-standalone.properties", "/usr/src/kafka-connect/kafka-connect-jmx/config/sample-jmx-configuration.properties"]

From 73ea9784ccd5363dde6ba7268721704b6db9bb7b Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 22:11:22 +0200
Subject: [PATCH 11/28] Each connect setup will have to be its own docker
 build...

as even without additional jars it needs properties files.
---
 build-contracts/docker-compose.yml            | 16 +++----
 connect-files/Dockerfile                      |  7 +++
 .../connect-file-source.properties.template   | 20 ++++++++
 connect-files/connect-files.sh                |  8 ++++
 connect-files/worker.properties               | 48 +++++++++++++++++++
 5 files changed, 91 insertions(+), 8 deletions(-)
 create mode 100644 connect-files/Dockerfile
 create mode 100644 connect-files/connect-file-source.properties.template
 create mode 100755 connect-files/connect-files.sh
 create mode 100644 connect-files/worker.properties

diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index 0247a09..3cd5734 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -58,17 +58,17 @@ services:
       - beginning
       - -e
 
-  test-kafka-connect:
-    build: ../kafka
-    labels:
-      com.yolean.build-contract: ""
+  connect-file-find:
+    build: ../connect-file-glob
     links:
       - kafka-0
-    entrypoint: ./bin/connect-standalone.sh
+    # set up a development container
+    entrypoint: /bin/sh
     command:
-      # TODO we need to configure bootstrap.servers=kafka-0:9092 here
-      - ./config/connect-standalone.properties
-      - ./config/connect-file-source.properties
+      - -exc
+      - tail -f /dev/null
+    volumes:
+      - ../connect-files:/usr/src/connect-files
 
   test-kafka-connect-build:
     build: ../kafka-connect-jmx
diff --git a/connect-files/Dockerfile b/connect-files/Dockerfile
new file mode 100644
index 0000000..42d0873
--- /dev/null
+++ b/connect-files/Dockerfile
@@ -0,0 +1,7 @@
+FROM solsson/kafka:0.11.0.0
+
+COPY worker.properties ./config/
+COPY connect-file-source.properties.template
+COPY connect-files.sh ./bin/
+
+ENV FILES_LIST_CMD="find /logs/ -name *.log"
diff --git a/connect-files/connect-file-source.properties.template b/connect-files/connect-file-source.properties.template
new file mode 100644
index 0000000..599cf4c
--- /dev/null
+++ b/connect-files/connect-file-source.properties.template
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name=local-file-source
+connector.class=FileStreamSource
+tasks.max=1
+file=test.txt
+topic=connect-test
\ No newline at end of file
diff --git a/connect-files/connect-files.sh b/connect-files/connect-files.sh
new file mode 100755
index 0000000..2563101
--- /dev/null
+++ b/connect-files/connect-files.sh
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+FILES=$($FILES_LIST_CMD)
+
+for F in $FILES; do
+  echo $F; 
+
+done
diff --git a/connect-files/worker.properties b/connect-files/worker.properties
new file mode 100644
index 0000000..0039796
--- /dev/null
+++ b/connect-files/worker.properties
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These are defaults. This file just demonstrates how to override some settings.
+bootstrap.servers=localhost:9092
+
+# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
+# need to configure these based on the format they want their data in when loaded from or stored into Kafka
+key.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
+# it to
+key.converter.schemas.enable=true
+value.converter.schemas.enable=true
+
+# The internal converter used for offsets and config data is configurable and must be specified, but most users will
+# always want to use the built-in default. Offset and config data is never visible outside of Kafka Connect in this format.
+internal.key.converter=org.apache.kafka.connect.json.JsonConverter
+internal.value.converter=org.apache.kafka.connect.json.JsonConverter
+internal.key.converter.schemas.enable=false
+internal.value.converter.schemas.enable=false
+
+offset.storage.file.filename=/tmp/connect.offsets
+# Flush much faster than normal, which is useful for testing/debugging
+offset.flush.interval.ms=10000
+
+# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
+# (connectors, converters, transformations). The list should consist of top level directories that include 
+# any combination of: 
+# a) directories immediately containing jars with plugins and their dependencies
+# b) uber-jars with plugins and their dependencies
+# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
+# Note: symlinks will be followed to discover dependencies or plugins.
+# Examples: 
+# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
+#plugin.path=

From 7b82d0f6d95b919dafd3c672104d6e744fa38616 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Thu, 20 Jul 2017 22:29:43 +0200
Subject: [PATCH 12/28] Quite useful log aggregation

---
 README.md                                     |  2 ++
 build-contracts/docker-compose.yml            | 27 +++++++++++++------
 connect-files/Dockerfile                      |  7 ++++-
 .../connect-file-source.properties.template   | 20 --------------
 connect-files/connect-files.sh                | 18 +++++++++++--
 connect-files/worker.properties               |  2 +-
 6 files changed, 44 insertions(+), 32 deletions(-)
 delete mode 100644 connect-files/connect-file-source.properties.template

diff --git a/README.md b/README.md
index 5df2e64..5dfa36a 100644
--- a/README.md
+++ b/README.md
@@ -27,4 +27,6 @@ $compose up -d zookeeper kafka-0
 $compose up test-topic-create-1
 $compose up test-snappy-compression
 $compose up test-consume-all
+$compose up --build -d consume-files
+$compose up test-consume-files
 ```
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index 3cd5734..6152ca6 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -9,6 +9,8 @@ services:
 
   kafka-0:
     build: ../kafka
+    links:
+      - zookeeper
     command:
       - config/server.properties
       - --override
@@ -58,17 +60,26 @@ services:
       - beginning
       - -e
 
-  connect-file-find:
-    build: ../connect-file-glob
+  connect-files:
+    build: ../connect-files
     links:
       - kafka-0
-    # set up a development container
-    entrypoint: /bin/sh
-    command:
-      - -exc
-      - tail -f /dev/null
+    # test with some real logs
     volumes:
-      - ../connect-files:/usr/src/connect-files
+      - /var/log:/logs
+
+  test-consume-files:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - -b
+      - kafka-0:9092
+      - -t
+      - files-000
+      - -C
+      - -o
+      - beginning
 
   test-kafka-connect-build:
     build: ../kafka-connect-jmx
diff --git a/connect-files/Dockerfile b/connect-files/Dockerfile
index 42d0873..cc549b8 100644
--- a/connect-files/Dockerfile
+++ b/connect-files/Dockerfile
@@ -1,7 +1,12 @@
 FROM solsson/kafka:0.11.0.0
 
 COPY worker.properties ./config/
-COPY connect-file-source.properties.template
 COPY connect-files.sh ./bin/
 
 ENV FILES_LIST_CMD="find /logs/ -name *.log"
+
+# Set up some sample logs
+RUN mkdir /logs/; \
+  echo "Mount /logs and/or change FILES_LIST_CMD (currently '$FILES_LIST_CMD') to read real content instead" > /logs/samplefile1.log;
+
+ENTRYPOINT ["./bin/connect-files.sh"]
diff --git a/connect-files/connect-file-source.properties.template b/connect-files/connect-file-source.properties.template
deleted file mode 100644
index 599cf4c..0000000
--- a/connect-files/connect-file-source.properties.template
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-name=local-file-source
-connector.class=FileStreamSource
-tasks.max=1
-file=test.txt
-topic=connect-test
\ No newline at end of file
diff --git a/connect-files/connect-files.sh b/connect-files/connect-files.sh
index 2563101..cf5007b 100755
--- a/connect-files/connect-files.sh
+++ b/connect-files/connect-files.sh
@@ -1,8 +1,22 @@
 #!/bin/bash
+set -e
 
 FILES=$($FILES_LIST_CMD)
 
-for F in $FILES; do
-  echo $F; 
+id=0
+connectors=""
+for FILE in $FILES; do
+  ((++id))
+  echo "$id: $FILE"
+  cat <<HERE > ./config/connect-file-source-$id.properties
+name=local-file-source-${id}
+connector.class=FileStreamSource
+tasks.max=1
+file=${FILE}
+topic=files-000
+HERE
 
+  connectors="$connectors ./config/connect-file-source-$id.properties"
 done
+
+./bin/connect-standalone.sh ./config/worker.properties $connectors
diff --git a/connect-files/worker.properties b/connect-files/worker.properties
index 0039796..ff76cd3 100644
--- a/connect-files/worker.properties
+++ b/connect-files/worker.properties
@@ -14,7 +14,7 @@
 # limitations under the License.
 
 # These are defaults. This file just demonstrates how to override some settings.
-bootstrap.servers=localhost:9092
+bootstrap.servers=kafka-0:9092
 
 # The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
 # need to configure these based on the format they want their data in when loaded from or stored into Kafka

From 845fafff77b2060589fa70ed9fa19d73e72140eb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 21 Jul 2017 17:18:24 +0200
Subject: [PATCH 13/28] worker.properties can basically be copied among connect
 builds

---
 connect-jmx/Dockerfile        | 22 ++++++++++++++++
 connect-jmx/worker.properties | 48 +++++++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 connect-jmx/Dockerfile
 create mode 100644 connect-jmx/worker.properties

diff --git a/connect-jmx/Dockerfile b/connect-jmx/Dockerfile
new file mode 100644
index 0000000..a36df75
--- /dev/null
+++ b/connect-jmx/Dockerfile
@@ -0,0 +1,22 @@
+FROM solsson/kafka:0.11.0.0
+
+# Note: this is an experimental build that isn't at all optimized to reduce size
+
+RUN apt-get update && apt-get install -y maven curl ca-certificates --no-install-recommends
+
+# TODO sort out where Zulu is, and/or tools.jar
+RUN apt-get update && apt-get install -y openjdk-8-jdk-headless --no-install-recommends
+
+RUN set -ex; \
+  mkdir /usr/src/kafka-connect; \
+  cd /usr/src/kafka-connect; \
+  curl -SLs https://github.com/srijiths/kafka-connectors/archive/dc0a7122650e697d3ae97c970a4785bbed949479.tar.gz \
+    | tar -xzf - --strip-components=1 -C ./; \
+  mvn clean install;
+
+RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar ./libs/
+
+COPY *.properties ./config/
+
+ENTRYPOINT ["./bin/connect-standalone.sh"]
+CMD ["./config/worker.properties", "./config/connect-jmx.properties"]
diff --git a/connect-jmx/worker.properties b/connect-jmx/worker.properties
new file mode 100644
index 0000000..ff76cd3
--- /dev/null
+++ b/connect-jmx/worker.properties
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# These are defaults. This file just demonstrates how to override some settings.
+bootstrap.servers=kafka-0:9092
+
+# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will
+# need to configure these based on the format they want their data in when loaded from or stored into Kafka
+key.converter=org.apache.kafka.connect.json.JsonConverter
+value.converter=org.apache.kafka.connect.json.JsonConverter
+# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply
+# it to
+key.converter.schemas.enable=true
+value.converter.schemas.enable=true
+
+# The internal converter used for offsets and config data is configurable and must be specified, but most users will
+# always want to use the built-in default. Offset and config data is never visible outside of Kafka Connect in this format.
+internal.key.converter=org.apache.kafka.connect.json.JsonConverter
+internal.value.converter=org.apache.kafka.connect.json.JsonConverter
+internal.key.converter.schemas.enable=false
+internal.value.converter.schemas.enable=false
+
+offset.storage.file.filename=/tmp/connect.offsets
+# Flush much faster than normal, which is useful for testing/debugging
+offset.flush.interval.ms=10000
+
+# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins
+# (connectors, converters, transformations). The list should consist of top level directories that include 
+# any combination of: 
+# a) directories immediately containing jars with plugins and their dependencies
+# b) uber-jars with plugins and their dependencies
+# c) directories immediately containing the package directory structure of classes of plugins and their dependencies
+# Note: symlinks will be followed to discover dependencies or plugins.
+# Examples: 
+# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors,
+#plugin.path=

From 50e0c9a993f0701a867ceba67f93d45b0b21cb48 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 21 Jul 2017 20:56:07 +0200
Subject: [PATCH 14/28] Demonstrates JMX to HTTP /metrics

---
 README.md                                     |  3 +-
 build-contracts/docker-compose.yml            | 38 ++++++++++++++++++-
 connect-jmx/connect-jmx.properties            | 19 ++++++++++
 kafka-connect-jmx/Dockerfile                  | 18 ---------
 .../Dockerfile                                |  0
 5 files changed, 57 insertions(+), 21 deletions(-)
 create mode 100644 connect-jmx/connect-jmx.properties
 delete mode 100644 kafka-connect-jmx/Dockerfile
 rename {kafka-prometheus-jmx-exporter => prometheus-jmx-exporter}/Dockerfile (100%)

diff --git a/README.md b/README.md
index 5dfa36a..d0c8e30 100644
--- a/README.md
+++ b/README.md
@@ -27,6 +27,7 @@ $compose up -d zookeeper kafka-0
 $compose up test-topic-create-1
 $compose up test-snappy-compression
 $compose up test-consume-all
-$compose up --build -d consume-files
+$compose up prometheus-jmx-exporter test-metrics-export
+$compose up --build -d connect-files
 $compose up test-consume-files
 ```
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index 6152ca6..eb76bda 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -11,6 +11,10 @@ services:
     build: ../kafka
     links:
       - zookeeper
+    environment:
+      - JMX_PORT=5555
+    expose:
+      - '5555'
     command:
       - config/server.properties
       - --override
@@ -19,6 +23,34 @@ services:
       - --override
       -   advertised.listener=PLAINTEXT://kafka-0:9092
 
+  prometheus-jmx-exporter:
+    #build: ../prometheus-jmx-exporter
+    image: solsson/kafka-prometheus-jmx-exporter
+    labels:
+      com.yolean.build-target: ""
+    links:
+      - kafka-0
+    # patch a config before start, as the image is designed for use with local JMX (same k8s pod)
+    entrypoint: /bin/bash
+    command:
+      - -c
+      - >
+        sed -i 's|127.0.0.1|kafka-0|' example_configs/kafka-prometheus-monitoring.yml;
+        cat example_configs/kafka-prometheus-monitoring.yml;
+        java -jar jmx_prometheus_httpserver.jar
+        5556 example_configs/kafka-prometheus-monitoring.yml
+
+  test-metrics-export:
+    image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874
+    command:
+      - --fail-early
+      - --retry
+      -   '10'
+      - --retry-delay
+      -   '3'
+      - --retry-connrefused
+      - http://prometheus-jmx-exporter:5556/metrics
+
   test-topic-create:
     build: ../kafka
     labels:
@@ -81,5 +113,7 @@ services:
       - -o
       - beginning
 
-  test-kafka-connect-build:
-    build: ../kafka-connect-jmx
+  connect-jmx:
+    build: ../connect-jmx
+    links:
+      - kafka-0
diff --git a/connect-jmx/connect-jmx.properties b/connect-jmx/connect-jmx.properties
new file mode 100644
index 0000000..23b7dd5
--- /dev/null
+++ b/connect-jmx/connect-jmx.properties
@@ -0,0 +1,19 @@
+# Name of the connecor
+name=jmx-source
+# Connector class to invoke the connector
+connector.class=com.sree.kafka.connectors.jmx.JmxConnector
+# Maximum number of tasks
+tasks.max=1
+
+# Kafka topic to push the messages
+kafka.topic=jmx-test
+# JMX is running for which application.
+# If you want JMX metrics for Kafka , then jmx.servicename=kafka
+# If you want JMX metrics for Flink , then jmx.servicename=flink etc..
+jmx.servicename=kafka
+# If jmx.servicename is kafka , then you have to provide zookeeper.host
+# Else zookeeper.host parameter is not required.
+zookeeper.host=zookeeper:2181
+# If jmx.servicename is not kafka , then below property is mandatory
+# Provide the full JMX URL separated by comma
+#jmx.url=54.238.221.37:8080,54.238.237.66:8080
diff --git a/kafka-connect-jmx/Dockerfile b/kafka-connect-jmx/Dockerfile
deleted file mode 100644
index 9c5f23c..0000000
--- a/kafka-connect-jmx/Dockerfile
+++ /dev/null
@@ -1,18 +0,0 @@
-FROM solsson/kafka:0.11.0.0
-
-RUN apt-get update && apt-get install -y maven curl ca-certificates --no-install-recommends
-
-# TODO sort out where Zulu is, and/or tools.jar
-RUN apt-get update && apt-get install -y openjdk-8-jdk-headless --no-install-recommends
-
-RUN set -ex; \
-  mkdir /usr/src/kafka-connect; \
-  cd /usr/src/kafka-connect; \
-  curl -SLs https://github.com/srijiths/kafka-connectors/archive/dc0a7122650e697d3ae97c970a4785bbed949479.tar.gz \
-    | tar -xzf - --strip-components=1 -C ./; \
-  mvn clean install;
-
-RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar ./libs/
-
-ENTRYPOINT ["./bin/connect-standalone.sh"]
-CMD ["./config/connect-standalone.properties", "/usr/src/kafka-connect/kafka-connect-jmx/config/sample-jmx-configuration.properties"]
diff --git a/kafka-prometheus-jmx-exporter/Dockerfile b/prometheus-jmx-exporter/Dockerfile
similarity index 100%
rename from kafka-prometheus-jmx-exporter/Dockerfile
rename to prometheus-jmx-exporter/Dockerfile

From 4b183a5b01d72b24f2cb4d88ea6d0b7a31ae86b6 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 21 Jul 2017 21:04:09 +0200
Subject: [PATCH 15/28] Demonstrates built connect jar, but messages are
 encoded json in outer json's payload

---
 README.md                          | 10 +++++++++-
 build-contracts/docker-compose.yml | 15 +++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index d0c8e30..7c5dca9 100644
--- a/README.md
+++ b/README.md
@@ -20,14 +20,22 @@ Build and test using: `docker run -v /var/run/docker.sock:/var/run/docker.sock -
 
 To keep kafka running for local use, uncomment `ports` 9092 and run: `docker-compose -f build-contracts/docker-compose.yml up --force-recreate`.
 
-While timing issues remain, start services individually
+While timing issues remain, start services individually...
+
 ```bash
 compose='docker-compose -f build-contracts/docker-compose.yml'
 $compose up -d zookeeper kafka-0
+$compose logs zookeeper kafka-0
+# can we create topics using the image's provided script?
 $compose up test-topic-create-1
+# can a producer send messages using snappy (has issues before with a class missing in the image)
 $compose up test-snappy-compression
 $compose up test-consume-all
+# demo the prometheus exporter image
 $compose up prometheus-jmx-exporter test-metrics-export
+# demo the log/file aggregation image
 $compose up --build -d connect-files
 $compose up test-consume-files
+# demo the JMX->kafka image
+$compose up --build connect-jmx test-jmx
 ```
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index eb76bda..be5a417 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -115,5 +115,20 @@ services:
 
   connect-jmx:
     build: ../connect-jmx
+    labels:
+      com.yolean.build-target: ""
     links:
       - kafka-0
+
+  test-jmx:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - -b
+      - kafka-0:9092
+      - -t
+      - jmx-test
+      - -C
+      - -o
+      - beginning

From 87d31727b7a20c71f0df326b6ca5e7581fa8f47c Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:02:37 +0200
Subject: [PATCH 16/28] Splits into separate docker-compose setups per area of
 interest

---
 README.md                                     | 19 +++--
 .../docker-compose.files-aggregation.yml      | 46 +++++++++++
 build-contracts/docker-compose.monitoring.yml | 74 ++++++++++++++++++
 build-contracts/docker-compose.yml            | 78 ++-----------------
 4 files changed, 139 insertions(+), 78 deletions(-)
 create mode 100644 build-contracts/docker-compose.files-aggregation.yml
 create mode 100644 build-contracts/docker-compose.monitoring.yml

diff --git a/README.md b/README.md
index 7c5dca9..7d65311 100644
--- a/README.md
+++ b/README.md
@@ -12,6 +12,18 @@ and related `kafka-` images under https://hub.docker.com/u/solsson/.
 
 These images are tested in production with https://github.com/Yolean/kubernetes-kafka/.
 
+## One image to rule them all
+
+Official [Kafka distributions](http://kafka.apache.org/downloads) contain startup scripts and config for various services and clients. Thus `./kafka` produces a multi-purpose image for direct use and specialized docker builds.
+
+### Example of downstream image: Kafka Connect
+
+See ./connect-jmx
+
+### Example downstream image: Kafka Streams
+
+TODO
+
 ## Building
 
 Rudimentary compliance with kubernetes-kafka is tested using a [build-contract](https://github.com/Yolean/build-contract/).
@@ -31,11 +43,8 @@ $compose up test-topic-create-1
 # can a producer send messages using snappy (has issues before with a class missing in the image)
 $compose up test-snappy-compression
 $compose up test-consume-all
-# demo the prometheus exporter image
-$compose up prometheus-jmx-exporter test-metrics-export
 # demo the log/file aggregation image
-$compose up --build -d connect-files
-$compose up test-consume-files
+docker-compose -f build-contracts/docker-compose.files-aggregation.yml up
 # demo the JMX->kafka image
-$compose up --build connect-jmx test-jmx
+docker-compose -f build-contracts/docker-compose.monitoring.yml up
 ```
diff --git a/build-contracts/docker-compose.files-aggregation.yml b/build-contracts/docker-compose.files-aggregation.yml
new file mode 100644
index 0000000..0b29e07
--- /dev/null
+++ b/build-contracts/docker-compose.files-aggregation.yml
@@ -0,0 +1,46 @@
+version: '2.0'
+services:
+
+  zookeeper:
+    build: ../kafka
+    entrypoint: ./bin/zookeeper-server-start.sh
+    command:
+      - config/zookeeper.properties
+
+  kafka-0:
+    build: ../kafka
+    links:
+      - zookeeper
+    command:
+      - config/server.properties
+      - --override
+      -   broker.id=0
+      - --override
+      -   advertised.listener=PLAINTEXT://kafka-0:9092
+
+  connect-files:
+    build: ../connect-files
+    labels:
+      com.yolean.build-target: ""
+    links:
+      - kafka-0
+
+  test-connect-files-real-logs:
+    build: ../connect-files
+    links:
+      - kafka-0
+    volumes:
+      - /var/log:/logs
+
+  test-consume-files:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - -b
+      - kafka-0:9092
+      - -t
+      - files-000
+      - -C
+      - -o
+      - beginning
diff --git a/build-contracts/docker-compose.monitoring.yml b/build-contracts/docker-compose.monitoring.yml
new file mode 100644
index 0000000..0668f58
--- /dev/null
+++ b/build-contracts/docker-compose.monitoring.yml
@@ -0,0 +1,74 @@
+version: '2.0'
+services:
+
+  zookeeper:
+    build: ../kafka
+    entrypoint: ./bin/zookeeper-server-start.sh
+    command:
+      - config/zookeeper.properties
+
+  kafka-0:
+    build: ../kafka
+    links:
+      - zookeeper
+    environment:
+      - JMX_PORT=5555
+    expose:
+      - '5555'
+    command:
+      - config/server.properties
+      - --override
+      -   broker.id=0
+      - --override
+      -   advertised.listener=PLAINTEXT://kafka-0:9092
+
+  prometheus-jmx-exporter:
+    build: ../prometheus-jmx-exporter
+    labels:
+      com.yolean.build-target: ""
+    links:
+      - kafka-0
+    # patch a config before start, as the image is designed for use with local JMX (same k8s pod)
+    entrypoint: /bin/bash
+    command:
+      - -c
+      - >
+        sed -i 's|127.0.0.1|kafka-0|' example_configs/kafka-prometheus-monitoring.yml;
+        cat example_configs/kafka-prometheus-monitoring.yml;
+        java -jar jmx_prometheus_httpserver.jar
+        5556 example_configs/kafka-prometheus-monitoring.yml
+
+  test-metrics-export:
+    image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - --fail-early
+      - --retry
+      -   '10'
+      - --retry-delay
+      -   '3'
+      - --retry-connrefused
+      - http://prometheus-jmx-exporter:5556/metrics
+
+  connect-jmx:
+    build: ../connect-jmx
+    labels:
+      com.yolean.build-target: ""
+    links:
+      - kafka-0
+
+  # TODO starts too fast, gets % KC_ERROR: Failed to query metadata for topic jmx-test: Local: Broker transport failure
+  # needs to retry until kafka+topic exists
+  test-jmx:
+    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
+    labels:
+      com.yolean.build-contract: ""
+    command:
+      - -b
+      - kafka-0:9092
+      - -t
+      - jmx-test
+      - -C
+      - -o
+      - beginning
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index be5a417..5e0629a 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -9,12 +9,11 @@ services:
 
   kafka-0:
     build: ../kafka
+    image: solsson/kafka
+    labels:
+      com.yolean.build-target: ""
     links:
       - zookeeper
-    environment:
-      - JMX_PORT=5555
-    expose:
-      - '5555'
     command:
       - config/server.properties
       - --override
@@ -23,34 +22,6 @@ services:
       - --override
       -   advertised.listener=PLAINTEXT://kafka-0:9092
 
-  prometheus-jmx-exporter:
-    #build: ../prometheus-jmx-exporter
-    image: solsson/kafka-prometheus-jmx-exporter
-    labels:
-      com.yolean.build-target: ""
-    links:
-      - kafka-0
-    # patch a config before start, as the image is designed for use with local JMX (same k8s pod)
-    entrypoint: /bin/bash
-    command:
-      - -c
-      - >
-        sed -i 's|127.0.0.1|kafka-0|' example_configs/kafka-prometheus-monitoring.yml;
-        cat example_configs/kafka-prometheus-monitoring.yml;
-        java -jar jmx_prometheus_httpserver.jar
-        5556 example_configs/kafka-prometheus-monitoring.yml
-
-  test-metrics-export:
-    image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874
-    command:
-      - --fail-early
-      - --retry
-      -   '10'
-      - --retry-delay
-      -   '3'
-      - --retry-connrefused
-      - http://prometheus-jmx-exporter:5556/metrics
-
   test-topic-create:
     build: ../kafka
     labels:
@@ -78,6 +49,8 @@ services:
       - -exc
       - sleep 5; echo "Message from $${HOSTNAME} at $$(date)" | kafkacat -z snappy -b kafka-0:9092 -t test1 -P
 
+  # TODO starts too fast, gets % KC_ERROR: Failed to query metadata for topic test1: Local: Broker transport failure
+  # needs to retry until kafka+topic exists
   test-consume-all:
     image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
     labels:
@@ -91,44 +64,3 @@ services:
       - -o
       - beginning
       - -e
-
-  connect-files:
-    build: ../connect-files
-    links:
-      - kafka-0
-    # test with some real logs
-    volumes:
-      - /var/log:/logs
-
-  test-consume-files:
-    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
-    labels:
-      com.yolean.build-contract: ""
-    command:
-      - -b
-      - kafka-0:9092
-      - -t
-      - files-000
-      - -C
-      - -o
-      - beginning
-
-  connect-jmx:
-    build: ../connect-jmx
-    labels:
-      com.yolean.build-target: ""
-    links:
-      - kafka-0
-
-  test-jmx:
-    image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8
-    labels:
-      com.yolean.build-contract: ""
-    command:
-      - -b
-      - kafka-0:9092
-      - -t
-      - jmx-test
-      - -C
-      - -o
-      - beginning

From 74ad3aac742f433d207ecd0fb6304d01007648cc Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:03:43 +0200
Subject: [PATCH 17/28] Now adds less than 6 MB to the official image

---
 connect-jmx/Dockerfile | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/connect-jmx/Dockerfile b/connect-jmx/Dockerfile
index a36df75..6e84a78 100644
--- a/connect-jmx/Dockerfile
+++ b/connect-jmx/Dockerfile
@@ -1,20 +1,27 @@
 FROM solsson/kafka:0.11.0.0
 
-# Note: this is an experimental build that isn't at all optimized to reduce size
-
-RUN apt-get update && apt-get install -y maven curl ca-certificates --no-install-recommends
-
-# TODO sort out where Zulu is, and/or tools.jar
-RUN apt-get update && apt-get install -y openjdk-8-jdk-headless --no-install-recommends
-
 RUN set -ex; \
-  mkdir /usr/src/kafka-connect; \
-  cd /usr/src/kafka-connect; \
+  buildDeps='curl ca-certificates'; \
+  apt-get update && apt-get install -y $buildDeps --no-install-recommends; \
+  \
+  MAVEN_VERSION=3.5.0 PATH=$PATH:$(pwd)/maven/bin; \
+  mkdir ./maven; \
+  curl -SLs https://archive.apache.org/dist/maven/maven-3/$MAVEN_VERSION/binaries/apache-maven-$MAVEN_VERSION-bin.tar.gz | tar -xzf - --strip-components=1 -C ./maven; \
+  mvn --version; \
+  \
+  mkdir ./kafka-connectors; \
+  cd ./kafka-connectors; \
   curl -SLs https://github.com/srijiths/kafka-connectors/archive/dc0a7122650e697d3ae97c970a4785bbed949479.tar.gz \
     | tar -xzf - --strip-components=1 -C ./; \
-  mvn clean install;
-
-RUN mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar ./libs/
+  mvn clean install; \
+  cd ..; \
+  mv ~/.m2/repository/com/sree/kafka/kafka-connect-jmx/0.0.1/kafka-connect-jmx-0.0.1-jar-with-dependencies.jar ./libs/; \
+  rm -rf ./kafka-connectors; \
+  rm -rf ./maven ~/.m2; \
+  \
+  apt-get purge -y --auto-remove $buildDeps; \
+  rm -rf /var/lib/apt/lists/*; \
+  rm /var/log/dpkg.log /var/log/apt/*.log
 
 COPY *.properties ./config/
 

From 415bb65f7b5a1ae2b14ea85088f9a30ba7011755 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:04:47 +0200
Subject: [PATCH 18/28] Clarifies that we use 3rd party source here

---
 connect-jmx/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/connect-jmx/Dockerfile b/connect-jmx/Dockerfile
index 6e84a78..fc134a3 100644
--- a/connect-jmx/Dockerfile
+++ b/connect-jmx/Dockerfile
@@ -1,5 +1,7 @@
 FROM solsson/kafka:0.11.0.0
 
+ENV srijiths-kafka-connectors-version=dc0a7122650e697d3ae97c970a4785bbed949479
+
 RUN set -ex; \
   buildDeps='curl ca-certificates'; \
   apt-get update && apt-get install -y $buildDeps --no-install-recommends; \
@@ -11,7 +13,7 @@ RUN set -ex; \
   \
   mkdir ./kafka-connectors; \
   cd ./kafka-connectors; \
-  curl -SLs https://github.com/srijiths/kafka-connectors/archive/dc0a7122650e697d3ae97c970a4785bbed949479.tar.gz \
+  curl -SLs https://github.com/srijiths/kafka-connectors/archive/$srijiths-kafka-connectors-version.tar.gz \
     | tar -xzf - --strip-components=1 -C ./; \
   mvn clean install; \
   cd ..; \

From e4de7e6f931244d15a11da74c1b6192a0c87e431 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:23:05 +0200
Subject: [PATCH 19/28] Honors the nature of Kafka distributions

---
 .../docker-compose.files-aggregation.yml          |  3 +++
 build-contracts/docker-compose.monitoring.yml     |  3 +++
 build-contracts/docker-compose.yml                |  3 +++
 kafka/Dockerfile                                  |  5 ++---
 kafka/docker-help.sh                              | 15 +++++++++++++++
 5 files changed, 26 insertions(+), 3 deletions(-)
 create mode 100755 kafka/docker-help.sh

diff --git a/build-contracts/docker-compose.files-aggregation.yml b/build-contracts/docker-compose.files-aggregation.yml
index 0b29e07..685de9c 100644
--- a/build-contracts/docker-compose.files-aggregation.yml
+++ b/build-contracts/docker-compose.files-aggregation.yml
@@ -11,9 +11,12 @@ services:
     build: ../kafka
     links:
       - zookeeper
+    entrypoint: ./bin/bin/kafka-server-start.sh
     command:
       - config/server.properties
       - --override
+      -   zookeeper.connect=zookeeper:2181
+      - --override
       -   broker.id=0
       - --override
       -   advertised.listener=PLAINTEXT://kafka-0:9092
diff --git a/build-contracts/docker-compose.monitoring.yml b/build-contracts/docker-compose.monitoring.yml
index 0668f58..5656880 100644
--- a/build-contracts/docker-compose.monitoring.yml
+++ b/build-contracts/docker-compose.monitoring.yml
@@ -15,9 +15,12 @@ services:
       - JMX_PORT=5555
     expose:
       - '5555'
+    entrypoint: ./bin/bin/kafka-server-start.sh
     command:
       - config/server.properties
       - --override
+      -   zookeeper.connect=zookeeper:2181
+      - --override
       -   broker.id=0
       - --override
       -   advertised.listener=PLAINTEXT://kafka-0:9092
diff --git a/build-contracts/docker-compose.yml b/build-contracts/docker-compose.yml
index 5e0629a..e1ec794 100644
--- a/build-contracts/docker-compose.yml
+++ b/build-contracts/docker-compose.yml
@@ -14,9 +14,12 @@ services:
       com.yolean.build-target: ""
     links:
       - zookeeper
+    entrypoint: ./bin/kafka-server-start.sh
     command:
       - config/server.properties
       - --override
+      -   zookeeper.connect=zookeeper:2181
+      - --override
       -   broker.id=0
       # unlike Kubernetes StatefulSet, compose gives containers a random hostname (leading to redirects to a hex name)
       - --override
diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index ba412e4..6cc4b6d 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -19,7 +19,6 @@ RUN set -ex; \
   rm /var/log/dpkg.log /var/log/apt/*.log
 
 WORKDIR /opt/kafka
-ENTRYPOINT ["bin/kafka-server-start.sh"]
 
-RUN sed -i 's/zookeeper.connect=localhost:2181/zookeeper.connect=zookeeper:2181/' config/server.properties
-CMD ["config/server.properties"]
+COPY docker-help.sh /usr/local/bin/docker-help
+ENTRYPOINT ["docker-help"]
diff --git a/kafka/docker-help.sh b/kafka/docker-help.sh
new file mode 100755
index 0000000..8c82eb7
--- /dev/null
+++ b/kafka/docker-help.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+echo "Hi,"
+echo ""
+echo "This image is basically just the official Kafka distribution,"
+echo "containing both servers and utils, each with its own help output."
+echo ""
+echo "Select as entrypoint one of these scripts:"
+find ./bin/ -name *.sh
+echo ""
+echo "You might find one of the sample config files useful:"
+find ./config/ -name *.properties
+echo ""
+echo "Add more using volumes, or downstream images."
+echo "Enjoy Kafka!"
+echo ""

From 385eff8821a735518a55dbad5d7e4caa978c1066 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:32:03 +0200
Subject: [PATCH 20/28] Tries to replace the old image behavior with
 instructions

---
 README.md | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 7d65311..e70bdde 100644
--- a/README.md
+++ b/README.md
@@ -1,21 +1,37 @@
-# dockerfiles
+# Kafka docker builds
+
+Automated [Kafka](http://kafka.apache.org/) builds for https://hub.docker.com/r/solsson/kafka/
+and related `kafka-` images under https://hub.docker.com/u/solsson/.
 
-Nowadays we're using separate repositories for dockerization projects.
+---
+
+This repo used to contain misc dockerfiles, but they've moved to separate repositories for dockerization projects.
+We've kept the repository name to avoid breaking the automated build of solsson/kafka in Docker Hub.
 
 For legacy Dockerfiles from this repo (if you navigated to here from a Docker Hub [solsson](https://hub.docker.com/u/solsson/) image),
 see https://github.com/solsson/dockerfiles/tree/misc-dockerfiles.
 
-# Kafka docker builds
+---
 
-This repository maintains automated [Kafka](http://kafka.apache.org/) builds for https://hub.docker.com/r/solsson/kafka/
-and related `kafka-` images under https://hub.docker.com/u/solsson/.
+Our kafka images are tested in production with https://github.com/Yolean/kubernetes-kafka/.
 
-These images are tested in production with https://github.com/Yolean/kubernetes-kafka/.
+You most likely need to mount your own config files, or for `./bin/kafka-server-start.sh` use overrides like:
+```
+  --override zookeeper.connect=zookeeper:2181
+  --override log.dirs=/var/lib/kafka/data/topics
+  --override log.retention.hours=-1
+  --override broker.id=0
+  --override advertised.listener=PLAINTEXT://kafka-0:9092
+```
 
 ## One image to rule them all
 
 Official [Kafka distributions](http://kafka.apache.org/downloads) contain startup scripts and config for various services and clients. Thus `./kafka` produces a multi-purpose image for direct use and specialized docker builds.
 
+We could build specialized images like `kafka-server` but we have two reasons not to:
+ * Won't be as transparent in Docker Hub because you can't use Automated Build without scripting.
+ * In reality you'll need to control your own config anyway.
+
 ### Example of downstream image: Kafka Connect
 
 See ./connect-jmx

From e21ae32f0bbcaff8ac499376ce14e7eaaf60f490 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 22 Jul 2017 13:40:04 +0200
Subject: [PATCH 21/28] Slightly shorter

---
 README.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index e70bdde..50f8bf5 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Kafka docker builds
 
-Automated [Kafka](http://kafka.apache.org/) builds for https://hub.docker.com/r/solsson/kafka/
+Automated [Kafka](http://kafka.apache.org/) builds for [solsson/kafka](https://hub.docker.com/r/solsson/kafka/)
 and related `kafka-` images under https://hub.docker.com/u/solsson/.
 
 ---
@@ -44,11 +44,7 @@ TODO
 
 Rudimentary compliance with kubernetes-kafka is tested using a [build-contract](https://github.com/Yolean/build-contract/).
 
-Build and test using: `docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source solsson/build-contract test`
-
-To keep kafka running for local use, uncomment `ports` 9092 and run: `docker-compose -f build-contracts/docker-compose.yml up --force-recreate`.
-
-While timing issues remain, start services individually...
+Build and test using: `docker run -v /var/run/docker.sock:/var/run/docker.sock -v $(pwd)/:/source solsson/build-contract test`. However... while timing issues remain you need some manual intervention:
 
 ```bash
 compose='docker-compose -f build-contracts/docker-compose.yml'

From 9978188a69cf4b04bfe2f4f3659fce1d92027ddb Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sun, 23 Jul 2017 06:23:29 +0200
Subject: [PATCH 22/28] Adds the nc command as it's useful for liveness probes

---
 kafka/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 6cc4b6d..71f5a5e 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -3,7 +3,7 @@ FROM debian:stable-20170620
 ENV KAFKA_VERSION=0.11.0.0 ZULU_OPENJDK_VERSION="8=8.21.0.1" KAFKA_BIN_SCALA_VERSION=2.11
 
 RUN set -ex; \
-  runDeps=''; \
+  runDeps='netcat-openbsd'; \
   buildDeps='curl ca-certificates gnupg dirmngr'; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
   \

From b34eb6c0fcd7f35c4344253337fd5dc3d56d9eb7 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 25 Jul 2017 19:42:08 +0200
Subject: [PATCH 23/28] Adds base image, as Zulu will be upgraded less
 frequently than Kafka

---
 jre-zulu/Dockerfile | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)
 create mode 100644 jre-zulu/Dockerfile

diff --git a/jre-zulu/Dockerfile b/jre-zulu/Dockerfile
new file mode 100644
index 0000000..fadfe0f
--- /dev/null
+++ b/jre-zulu/Dockerfile
@@ -0,0 +1,19 @@
+FROM debian:stable-20170620
+
+ENV ZULU_OPENJDK_VERSION="8=8.21.0.1"
+
+RUN set -ex; \
+  runDeps=''; \
+  buildDeps='gnupg dirmngr'; \
+  apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
+  \
+  apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9; \
+  echo 'deb http://repos.azulsystems.com/debian stable main' > /etc/apt/sources.list.d/zulu.list; \
+  apt-get update && apt-get install -y zulu-${ZULU_OPENJDK_VERSION} --no-install-recommends; \
+  \
+  cd /usr/lib/jvm/zulu-8-amd64/; \
+  rm -rf demo man sample src.zip; \
+  \
+  apt-get purge -y --auto-remove $buildDeps; \
+  rm -rf /var/lib/apt/lists/*; \
+  rm -rf /var/log/dpkg.log /var/log/alternatives.log /var/log/apt

From 15ba474f2e610326880204fd69851919939684d4 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 25 Jul 2017 19:49:58 +0200
Subject: [PATCH 24/28] Builds the kafka image from the base jre image

---
 kafka/Dockerfile | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 71f5a5e..82a2fb0 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,22 +1,18 @@
-FROM debian:stable-20170620
+FROM solsson/kafka-jre
 
-ENV KAFKA_VERSION=0.11.0.0 ZULU_OPENJDK_VERSION="8=8.21.0.1" KAFKA_BIN_SCALA_VERSION=2.11
+ENV KAFKA_VERSION=0.11.0.0 KAFKA_BIN_SCALA_VERSION=2.11
 
 RUN set -ex; \
   runDeps='netcat-openbsd'; \
-  buildDeps='curl ca-certificates gnupg dirmngr'; \
+  buildDeps='curl ca-certificates'; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
   \
-  apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 0x219BD9C9; \
-  echo 'deb http://repos.azulsystems.com/debian stable main' > /etc/apt/sources.list.d/zulu.list; \
-  apt-get update && apt-get install -y zulu-${ZULU_OPENJDK_VERSION} --no-install-recommends; \
-  \
   mkdir -p /opt/kafka; \
   curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$KAFKA_BIN_SCALA_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \
   apt-get purge -y --auto-remove $buildDeps; \
   rm -rf /var/lib/apt/lists/*; \
-  rm /var/log/dpkg.log /var/log/apt/*.log
+  rm -rf /var/log/dpkg.log /var/log/alternatives.log /var/log/apt
 
 WORKDIR /opt/kafka
 

From 17f0f6ec56a618d04759c54e2f355fd3f4c42ca0 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 25 Jul 2017 19:51:42 +0200
Subject: [PATCH 25/28] Uses a specific base image build. 58 MB download on top
 of debian.

---
 kafka/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 82a2fb0..bad5333 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,4 +1,5 @@
-FROM solsson/kafka-jre
+# The only assumption we make about this FROM is that it has a JRE in path
+FROM solsson/kafka-jre@sha256:7765513cf5fa455a672a06f584058c1c81cc0b3b56cc56b0cfdf1a917a183f26
 
 ENV KAFKA_VERSION=0.11.0.0 KAFKA_BIN_SCALA_VERSION=2.11
 

From 08ee3b6d632937c6d4cc9a78633cc7931d133a03 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 25 Jul 2017 20:18:21 +0200
Subject: [PATCH 26/28] Changes scala env names to match assumptions in
 kafka-run-class.sh

---
 kafka/Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index bad5333..2b58c6e 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,7 +1,7 @@
 # The only assumption we make about this FROM is that it has a JRE in path
 FROM solsson/kafka-jre@sha256:7765513cf5fa455a672a06f584058c1c81cc0b3b56cc56b0cfdf1a917a183f26
 
-ENV KAFKA_VERSION=0.11.0.0 KAFKA_BIN_SCALA_VERSION=2.11
+ENV KAFKA_VERSION=0.11.0.0 SCALA_VERSION=2.11 SCALA_BINARY_VERSION=2.11
 
 RUN set -ex; \
   runDeps='netcat-openbsd'; \
@@ -9,7 +9,7 @@ RUN set -ex; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
   \
   mkdir -p /opt/kafka; \
-  curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$KAFKA_BIN_SCALA_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
+  curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$SCALA_BINARY_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \
   apt-get purge -y --auto-remove $buildDeps; \
   rm -rf /var/lib/apt/lists/*; \

From e7bd905effb12ee772000629fff95bb0fa8c9b1a Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Tue, 25 Jul 2017 20:20:32 +0200
Subject: [PATCH 27/28] Reflects kafka-run-class.sh better. We don't have
 Scala, neither at install nor runtime.

---
 kafka/Dockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index 2b58c6e..d7bce83 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -1,13 +1,14 @@
 # The only assumption we make about this FROM is that it has a JRE in path
 FROM solsson/kafka-jre@sha256:7765513cf5fa455a672a06f584058c1c81cc0b3b56cc56b0cfdf1a917a183f26
 
-ENV KAFKA_VERSION=0.11.0.0 SCALA_VERSION=2.11 SCALA_BINARY_VERSION=2.11
+ENV KAFKA_VERSION=0.11.0.0 SCALA_VERSION=2.11
 
 RUN set -ex; \
   runDeps='netcat-openbsd'; \
   buildDeps='curl ca-certificates'; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
   \
+  SCALA_BINARY_VERSION=$(echo $SCALA_VERSION | cut -f 1-2 -d '.'); \
   mkdir -p /opt/kafka; \
   curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$SCALA_BINARY_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \

From 03140809ca84ead66014fcdad68443b96ea258ca Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Wed, 26 Jul 2017 05:05:36 +0200
Subject: [PATCH 28/28] Slightly cleaner installation

---
 kafka/Dockerfile | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/kafka/Dockerfile b/kafka/Dockerfile
index d7bce83..238a812 100644
--- a/kafka/Dockerfile
+++ b/kafka/Dockerfile
@@ -4,6 +4,7 @@ FROM solsson/kafka-jre@sha256:7765513cf5fa455a672a06f584058c1c81cc0b3b56cc56b0cf
 ENV KAFKA_VERSION=0.11.0.0 SCALA_VERSION=2.11
 
 RUN set -ex; \
+  export DEBIAN_FRONTEND=noninteractive; \
   runDeps='netcat-openbsd'; \
   buildDeps='curl ca-certificates'; \
   apt-get update && apt-get install -y $runDeps $buildDeps --no-install-recommends; \
@@ -12,6 +13,8 @@ RUN set -ex; \
   mkdir -p /opt/kafka; \
   curl -SLs "https://www-eu.apache.org/dist/kafka/$KAFKA_VERSION/kafka_$SCALA_BINARY_VERSION-$KAFKA_VERSION.tgz" | tar -xzf - --strip-components=1 -C /opt/kafka; \
   \
+  rm -rf /opt/kafka/site-docs; \
+  \
   apt-get purge -y --auto-remove $buildDeps; \
   rm -rf /var/lib/apt/lists/*; \
   rm -rf /var/log/dpkg.log /var/log/alternatives.log /var/log/apt