From 09f34c7af920f7e8e6ebd93eba04dc84d69d2361 Mon Sep 17 00:00:00 2001 From: Hart Simha Date: Thu, 27 Apr 2017 18:33:16 -0700 Subject: [PATCH] Modify Makefile and Dockerfiles to speed up builds Build a base docker image for each platform with the requisite go executable, libvirt packages, and docker-machine source. When making releases, this base image will be used to build a second temporary image in which the source directory is mounted. This image is then used to launch a container in which the executable is built, after which both container and build image are removed. This speeds up compilation of the executables for each platform and allows for testing changes on all platforms more quickly --- .dockerignore | 4 ++ .gitignore | 1 + Makefile | 41 +++++++++++-------- .../Dockerfile.alpine3.4 | 8 +--- .../Dockerfile.alpine3.5 | 8 +--- .../Dockerfile.centos7 | 7 ---- dockerfiles/Dockerfile.opensuse42.2 | 12 ++++++ .../Dockerfile.ubuntu14.04 | 7 ---- .../Dockerfile.ubuntu16.04 | 7 ---- make/base | 32 +++++++++++++++ make/build | 24 +++++++++++ out/.gitkeep | 0 12 files changed, 100 insertions(+), 51 deletions(-) create mode 100644 .dockerignore rename Dockerfile.alpine3.4 => dockerfiles/Dockerfile.alpine3.4 (60%) rename Dockerfile.alpine3.5 => dockerfiles/Dockerfile.alpine3.5 (60%) rename Dockerfile.centos7 => dockerfiles/Dockerfile.centos7 (58%) create mode 100644 dockerfiles/Dockerfile.opensuse42.2 rename Dockerfile.ubuntu14.04 => dockerfiles/Dockerfile.ubuntu14.04 (59%) rename Dockerfile.ubuntu16.04 => dockerfiles/Dockerfile.ubuntu16.04 (59%) create mode 100755 make/base create mode 100755 make/build create mode 100644 out/.gitkeep diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000000..69905cc17d8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.dockerignore +out +dockerfiles +make diff --git a/.gitignore b/.gitignore index 96554d125e7..34e223a61ec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ cmd/docker-machine-driver-kvm/docker-machine-driver-kvm *.sw* docker-machine-driver-kvm-* +out diff --git a/Makefile b/Makefile index ce5d680c526..b7c95d9b2dc 100644 --- a/Makefile +++ b/Makefile @@ -1,36 +1,41 @@ -PREFIX=docker-machine-driver-kvm -MACHINE_VERSION=v0.10.0 -GO_VERSION=1.8.1 +export PREFIX=docker-machine-driver-kvm +export GO_VERSION ?= 1.8.1 +export MACHINE_VERSION ?= v0.10.0 +export GIT_ROOT=$(shell git rev-parse --show-toplevel) DESCRIBE=$(shell git describe --tags) -TARGETS=$(addprefix $(PREFIX)-, alpine3.4 alpine3.5 ubuntu14.04 ubuntu16.04 centos7) +TARGETS=$(addprefix out/$(PREFIX)-, alpine3.4 alpine3.5 ubuntu14.04 ubuntu16.04 centos7 opensuse42.2) build: $(TARGETS) -$(PREFIX)-%: Dockerfile.% - docker rmi -f $@ >/dev/null 2>&1 || true - docker rm -f $@-extract > /dev/null 2>&1 || true - echo "Building binaries for $@" - docker build --build-arg "MACHINE_VERSION=$(MACHINE_VERSION)" --build-arg "GO_VERSION=$(GO_VERSION)" -t $@ -f $< . - docker create --name $@-extract $@ sh - docker cp $@-extract:/go/bin/docker-machine-driver-kvm ./ - mv ./docker-machine-driver-kvm ./$@ - docker rm $@-extract || true - docker rmi $@ || true +$(PREFIX)-base\:%: dockerfiles/Dockerfile.% + "$(GIT_ROOT)/make/base" $@ + +Dockerfile.%.build: + @echo "FROM $(PREFIX)-base:$*" >> $@ + @echo "ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/go/bin" >> $@ + @echo "ENV GOPATH /go" >> $@ + @echo "COPY . /go/src/github.com/dhiltgen/docker-machine-kvm" >> $@ + +out/$(PREFIX)-%: $(PREFIX)-base\:% Dockerfile.%.build + "$(GIT_ROOT)/make/build" $< clean: - rm -f ./$(PREFIX)-* + rm -f ./out/$(PREFIX)-* +clean-bases: + docker rmi $(docker images -q $(PREFIX)-base) release: build @echo "Paste the following into the release page on github and upload the binaries..." @echo "" - @for bin in $(PREFIX)-* ; do \ + @for bin in out/$(PREFIX)-* ; do \ + bin=$$(basename "$$bin"); \ target=$$(echo $${bin} | cut -f5- -d-) ; \ - md5=$$(md5sum $${bin}) ; \ + md5=$$(md5sum out/$${bin}) ; \ echo "* $${target} - md5: $${md5}" ; \ echo '```' ; \ - echo " curl -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/$(DESCRIBE)/$${bin} > /usr/local/bin/$(PREFIX) \\ " ; \ + echo " curl -L https://github.com/dhiltgen/docker-machine-kvm/releases/download/$(DESCRIBE)/$${bin} > /usr/local/bin/$(PREFIX)" ; \ echo " chmod +x /usr/local/bin/$(PREFIX)" ; \ echo '```' ; \ done diff --git a/Dockerfile.alpine3.4 b/dockerfiles/Dockerfile.alpine3.4 similarity index 60% rename from Dockerfile.alpine3.4 rename to dockerfiles/Dockerfile.alpine3.4 index afa00740296..aa439e2c281 100644 --- a/Dockerfile.alpine3.4 +++ b/dockerfiles/Dockerfile.alpine3.4 @@ -3,13 +3,9 @@ FROM alpine:3.4 MAINTAINER Daniel Hiltgen ARG MACHINE_VERSION +# Uncomment the following line to build with docker engine < 1.13 +# ARG GO_VERSION ENV GOPATH /go RUN apk -v add --update libvirt-dev curl go git musl-dev gcc RUN git clone --branch ${MACHINE_VERSION} https://github.com/docker/machine.git /go/src/github.com/docker/machine - -COPY . /go/src/github.com/dhiltgen/docker-machine-kvm -WORKDIR /go/src/github.com/dhiltgen/docker-machine-kvm -RUN go get -v -d ./... - -RUN go install -v ./cmd/docker-machine-driver-kvm diff --git a/Dockerfile.alpine3.5 b/dockerfiles/Dockerfile.alpine3.5 similarity index 60% rename from Dockerfile.alpine3.5 rename to dockerfiles/Dockerfile.alpine3.5 index 1f42b0e62aa..b81a21e8f24 100644 --- a/Dockerfile.alpine3.5 +++ b/dockerfiles/Dockerfile.alpine3.5 @@ -3,13 +3,9 @@ FROM alpine:3.5 MAINTAINER Daniel Hiltgen ARG MACHINE_VERSION +# Uncomment the following line to build with docker engine < 1.13 +# ARG GO_VERSION ENV GOPATH /go RUN apk -v add --update libvirt-dev curl go git musl-dev gcc RUN git clone --branch ${MACHINE_VERSION} https://github.com/docker/machine.git /go/src/github.com/docker/machine - -COPY . /go/src/github.com/dhiltgen/docker-machine-kvm -WORKDIR /go/src/github.com/dhiltgen/docker-machine-kvm -RUN go get -v -d ./... - -RUN go install -v ./cmd/docker-machine-driver-kvm diff --git a/Dockerfile.centos7 b/dockerfiles/Dockerfile.centos7 similarity index 58% rename from Dockerfile.centos7 rename to dockerfiles/Dockerfile.centos7 index 693d364ee56..d163b2c4a2e 100644 --- a/Dockerfile.centos7 +++ b/dockerfiles/Dockerfile.centos7 @@ -8,11 +8,4 @@ ENV GOPATH /go RUN yum install -y libvirt-devel curl git gcc RUN curl -sSL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xzf - -ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/go/bin RUN git clone --branch ${MACHINE_VERSION} https://github.com/docker/machine.git /go/src/github.com/docker/machine - -COPY . /go/src/github.com/dhiltgen/docker-machine-kvm -WORKDIR /go/src/github.com/dhiltgen/docker-machine-kvm -RUN go get -v -d ./... - -RUN go install -v ./cmd/docker-machine-driver-kvm diff --git a/dockerfiles/Dockerfile.opensuse42.2 b/dockerfiles/Dockerfile.opensuse42.2 new file mode 100644 index 00000000000..ecae6466540 --- /dev/null +++ b/dockerfiles/Dockerfile.opensuse42.2 @@ -0,0 +1,12 @@ +FROM opensuse:42.2 + +LABEL maintainer "Hart Simha " +RUN zypper -n in libvirt-devel curl git gcc tar && \ + git clone https://github.com/docker/machine.git /go/src/github.com/docker/machine + +ARG GO_VERSION +RUN curl -sSL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xzf - + +ARG MACHINE_VERSION +WORKDIR /go/src/github.com/docker/machine +RUN git checkout $MACHINE_VERSION diff --git a/Dockerfile.ubuntu14.04 b/dockerfiles/Dockerfile.ubuntu14.04 similarity index 59% rename from Dockerfile.ubuntu14.04 rename to dockerfiles/Dockerfile.ubuntu14.04 index 3b1c1cc552d..68dc458a043 100644 --- a/Dockerfile.ubuntu14.04 +++ b/dockerfiles/Dockerfile.ubuntu14.04 @@ -8,11 +8,4 @@ ENV GOPATH /go RUN apt-get update && apt-get install -y libvirt-dev curl git gcc RUN curl -sSL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xzf - -ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/go/bin RUN git clone --branch ${MACHINE_VERSION} https://github.com/docker/machine.git /go/src/github.com/docker/machine - -COPY . /go/src/github.com/dhiltgen/docker-machine-kvm -WORKDIR /go/src/github.com/dhiltgen/docker-machine-kvm -RUN go get -v -d ./... - -RUN go install -v ./cmd/docker-machine-driver-kvm diff --git a/Dockerfile.ubuntu16.04 b/dockerfiles/Dockerfile.ubuntu16.04 similarity index 59% rename from Dockerfile.ubuntu16.04 rename to dockerfiles/Dockerfile.ubuntu16.04 index c6286daaa70..cf8c6d3d04a 100644 --- a/Dockerfile.ubuntu16.04 +++ b/dockerfiles/Dockerfile.ubuntu16.04 @@ -8,11 +8,4 @@ ENV GOPATH /go RUN apt-get update && apt-get install -y libvirt-dev curl git gcc RUN curl -sSL https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz | tar -C /usr/local -xzf - -ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin:/go/bin RUN git clone --branch ${MACHINE_VERSION} https://github.com/docker/machine.git /go/src/github.com/docker/machine - -COPY . /go/src/github.com/dhiltgen/docker-machine-kvm -WORKDIR /go/src/github.com/dhiltgen/docker-machine-kvm -RUN go get -v -d ./... - -RUN go install -v ./cmd/docker-machine-driver-kvm diff --git a/make/base b/make/base new file mode 100755 index 00000000000..941ebc149de --- /dev/null +++ b/make/base @@ -0,0 +1,32 @@ +#!/usr/bin/env sh + +# This script creates the platform-specific base image used as the FROM base for constructing +# the image which will be used to generate the docker-machine-driver-kvm-$platform binary. +# It expects the base image name (taking the form of $PREFIX-base:$PLATFORM) to be passed as an argument +set -e +GO_VERSION_ARG=GO_VERSION=${GO_VERSION:-1.8.1} +MACHINE_VERSION_ARG=MACHINE_VERSION=${MACHINE_VERSION:-v0.10.0} +GIT_ROOT=${GIT_ROOT:-$(git rev-parse --show-toplevel)} + +# Extract values 'BASE_IMAGE_NAME:PLATFORM' from the image name argument +BASE_IMAGE_NAME=${1%:*} +PLATFORM=${1#*:} + +# If the base image for this platform, go version, and machine version is not already present, build it. +# Labels are used to track the GO_VERSION and MACHINE_VERSION +if [ -z "$(docker images -q $@ -f label=$GO_VERSION_ARG -f label=$MACHINE_VERSION_ARG)" ]; then + # If an image with this BASE_IMAGE_NAME:PLATFORM already exists, store its ID for deletion later + PREVIOUS_IMAGE=$(docker images -q $@) + # Build the new image before deleting any previous one, to reuse common layers + docker build \ + --label $GO_VERSION_ARG --build-arg $GO_VERSION_ARG \ + --label $MACHINE_VERSION_ARG --build-arg $MACHINE_VERSION_ARG \ + -t $@ \ + -f "${GIT_ROOT}/dockerfiles/Dockerfile.${PLATFORM}" \ + "${GIT_ROOT}" + if [ -n "$PREVIOUS_IMAGE" ]; then + docker rmi $PREVIOUS_IMAGE + fi +else + echo "Image '$@' with labels $GO_VERSION_ARG $MACHINE_VERSION_ARG already exists" +fi diff --git a/make/build b/make/build new file mode 100755 index 00000000000..982b4d64f34 --- /dev/null +++ b/make/build @@ -0,0 +1,24 @@ +#!/usr/bin/env sh + +# This script builds the docker-machine-driver-kvm binary by first building a temporary +# image with the relevant docker-machine-kvm code inside, and PATH and GO_PATH set. The binary +# is then built in a container from this image and into 'out/'. Finally, the temporary +# image and container (with suffixes '-build' and '-extract) are deleted + +set -e +PREFIX=${PREFIX:-docker-machine-driver-kvm} +GIT_ROOT=${GIT_ROOT:-$(git rev-parse --show-toplevel)} + +BASE_IMAGE=$@ +PLATFORM=${BASE_IMAGE#*:} +BUILD_TARGET=$PREFIX-$PLATFORM +BUILD_IMAGE=$PREFIX-build:$PLATFORM + +echo "Building compilation image for out/$PREFIX-$PLATFORM" +docker build -t $BUILD_IMAGE -f "${GIT_ROOT}/Dockerfile.$PLATFORM.build" "${GIT_ROOT}" + +echo "Building binaries for out/$PREFIX-$PLATFORM" +docker run --name $BUILD_TARGET-extract $BUILD_IMAGE sh -c "cd /go/src/github.com/dhiltgen/docker-machine-kvm && go get -v -d ./... && go install -v ./cmd/docker-machine-driver-kvm" +docker cp $BUILD_TARGET-extract:/go/bin/docker-machine-driver-kvm out/$BUILD_TARGET +docker rm $BUILD_TARGET-extract +docker rmi $BUILD_IMAGE diff --git a/out/.gitkeep b/out/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d