Add Docker image for kettle.

kettle/.dockerignore

@@ -0,0 +1,2 @@
+build.db
+

kettle/Dockerfile

@@ -0,0 +1,20 @@
+FROM pypy:2
+
+ADD requirements.txt /kettle/
+RUN pip install -r /kettle/requirements.txt
+
+RUN apt-get update && apt-get install pv time sqlite3 && apt-get clean
+
+RUN curl -o installer https://sdk.cloud.google.com && bash installer --disable-prompts --install-dir=/ && rm installer && ln -s /google-cloud-sdk/bin/* /bin/
+
+RUN ln -s /data/build.db /kettle/
+
+# stupid hack to deal with debian python2 getsitepackages vs pypy's version
+RUN bash -c 'touch /usr/local/site-packages/google/{__init__.py,cloud/__init__.py,cloud/pubsub/__init__.py}'
+
+ADD *.py schema.json runner.sh /kettle/
+ADD buckets.yaml /
+
+ENV PYTHONPATH /usr/local/site-packages
+CMD ["/kettle/runner.sh"]
+VOLUME ["/data"]

kettle/Makefile

@@ -0,0 +1,33 @@
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+IMG = gcr.io/k8s-testimages/kettle
+TAG = $(shell date +v%Y%m%d)-$(shell git describe --tags --always --dirty)
+
+build:
+	cp ../buckets.yaml .
+	docker build -t $(IMG):$(TAG) .
+	rm buckets.yaml
+	docker tag $(IMG):$(TAG) $(IMG):latest
+	echo Built $(IMG):$(TAG) and tagged with $(IMG):latest ;
+
+push: build
+	gcloud docker -- push $(IMG):$(TAG)
+	gcloud docker -- push $(IMG):latest
+	echo Pushed $(IMG):latest and $(IMG):$(TAG)
+
+
+.PHONY: all build push
+
+all: build

kettle/requirements.txt

@@ -1,3 +1,4 @@
+requests
 google-cloud-bigquery
 google-cloud-pubsub
 pyyaml

kettle/runner.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+# Copyright 2016 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# A wrapper script for running kettle
+
+# Update gcloud
+gcloud components update
+
+# Authenticate gcloud
+if [[ -n "${GOOGLE_APPLICATION_CREDENTIALS:-}" ]]; then
+  gcloud auth activate-service-account --key-file="${GOOGLE_APPLICATION_CREDENTIALS}"
+fi
+
+bq show <<< $'\n' > /dev/null  # create initial bq config
+
+while true; do
+  /kettle/update.py
+done