build/ci: fix it (#209)

Did some refactoring so that the correct version is passed into the kubernetes build, among other things.
rancher · Aug 20, 2020 · b4d0ad8 · b4d0ad8
1 parent 22a6e8b
commit b4d0ad8
Show file tree

Hide file tree

Showing 42 changed files with 345 additions and 266 deletions.
diff --git a/.drone.yml b/.drone.yml
@@ -36,13 +36,13 @@ steps:
     event:
     - tag
 
-- name: docker-k8s-build
+- name: build-image-kubernetes
   image: rancher/build-base:v1.14.2
   volumes:
   - name: docker
     path: /var/run/docker.sock
   commands:
-  - DRONE_TAG=${DRONE_TAG} BUILD_K8S_IMAGE=true make k8s-image
+  - DRONE_TAG=${DRONE_TAG} BUILD_K8S_IMAGE=true make build-image-kubernetes
   environment:
     DOCKER_USERNAME:
       from_secret: docker_username
@@ -52,14 +52,14 @@ steps:
     instance:
     - drone-pr.rancher.io
 
-- name: docker-k8s-publish
+- name: publish-image-kubernetes
   image: rancher/build-base:v1.14.2
   volumes:
     - name: docker
       path: /var/run/docker.sock
   commands:
     - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
-    - DRONE_TAG=${DRONE_TAG} BUILD_K8S_IMAGE=true make k8s-image-publish
+    - DRONE_TAG=${DRONE_TAG} make publish-image-kubernetes
   environment:
     DOCKER_USERNAME:
       from_secret: docker_username
@@ -74,7 +74,7 @@ steps:
     event:
     - tag
 
-- name: docker-publish
+- name: publish-image-runtime
   image: rancher/build-base:v1.14.2
   volumes:
     - name: docker
@@ -86,7 +86,7 @@ steps:
       from_secret: docker_password
   commands:
     - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
-    - DRONE_TAG=${DRONE_TAG} make image-publish
+    - DRONE_TAG=${DRONE_TAG} make publish-image-runtime
   when:
     instance:
     - drone-publish.rancher.io
@@ -110,7 +110,7 @@ platform:
   arch: amd64
 
 steps:
-- name: manifest
+- name: manifest-runtime
   image: rancher/build-base:v1.14.2
   volumes:
     - name: docker
@@ -122,7 +122,28 @@ steps:
       from_secret: docker_password
   commands:
     - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
-    - DRONE_TAG=${DRONE_TAG} make image-manifest
+    - DRONE_TAG=${DRONE_TAG} make publish-manifest-runtime
+  when:
+    instance:
+    - drone-publish.rancher.io
+    ref:
+    - refs/head/master
+    - refs/tags/*
+    event:
+    - tag
+- name: manifest-kubernetes
+  image: rancher/build-base:v1.14.2
+  volumes:
+  - name: docker
+    path: /var/run/docker.sock
+  environment:
+    DOCKER_USERNAME:
+      from_secret: docker_username
+    DOCKER_PASSWORD:
+      from_secret: docker_password
+  commands:
+  - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD
+  - DRONE_TAG=${DRONE_TAG} make publish-manifest-kubernetes
   when:
     instance:
     - drone-publish.rancher.io

diff --git a/BUILDING.md b/BUILDING.md
@@ -40,8 +40,8 @@ has been bind-mounted into the container ready to be imported into containerd on
 To run the built artifact(s) locally or on a remote host:
 - install prerequisites mentioned above
 - copy `./bin/rke2` to the path on your host
-- copy `./build/images/rke2-runtime-image-$(go env GOARCH).tar` to `/var/lib/rancher/rke2/agent/images/` on your host
-- if testing airgap, also copy `./build/images/airgap.tar` to `/var/lib/rancher/rke2/agent/images/` on your host
+- copy `./build/images/rke2-runtime.tar` to `/var/lib/rancher/rke2/agent/images/` on your host
+- if testing airgap, also copy `./build/images/rke2-airgap.tar` to `/var/lib/rancher/rke2/agent/images/` on your host
 - run rke2 server: `rke2 server --token=test`
 
 ### kubectl

diff --git a/Dockerfile b/Dockerfile
@@ -1,12 +1,30 @@
 ARG KUBERNETES_VERSION=dev
 # Build environment
 FROM rancher/build-base:v1.14.2 AS build
-# Yep nothing special here yet
-
+RUN set -x \
+ && export DEBIAN_FRONTEND=noninteractive \
+ && apt-get -y update \
+ && apt-get -y install \
+    libseccomp-dev \
+    rsync
 # Shell used for debugging
 FROM build AS shell
-RUN apt-get update && \
-    apt-get install -y iptables socat vim bash-completion less psmisc libseccomp-dev sudo
+RUN set -x \
+ && export DEBIAN_FRONTEND=noninteractive \
+ && apt-get -y update \
+ && apt-get -y install \
+    bash \
+    bash-completion \
+    git \
+    iptables \
+    less \
+    libseccomp2 \
+    psmisc \
+    rsync \
+    seccomp \
+    socat \
+    sudo \
+    vim
 RUN cp -f /etc/skel/.bashrc /etc/skel/.profile /root/ && \
     echo 'alias abort="echo -e '\''q\ny\n'\'' | ./bin/dlv connect :2345"' >> /root/.bashrc
 ENV PATH=/var/lib/rancher/rke2/bin:$PATH
@@ -18,7 +36,7 @@ VOLUME /var/lib/rancher/k3s
 # Dapper/Drone/CI environment
 FROM build AS dapper
 
-ENV DAPPER_ENV GODEBUG REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION
+ENV DAPPER_ENV GODEBUG REPO TAG DRONE_TAG PAT_USERNAME PAT_TOKEN KUBERNETES_VERSION DOCKER_BUILDKIT
 ENV DAPPER_OUTPUT ./dist ./bin ./build
 ENV DAPPER_DOCKER_SOCKET true
 ENV DAPPER_TARGET dapper
@@ -27,19 +45,30 @@ RUN curl -sL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh
 WORKDIR /source
 # End Dapper stuff
 
+FROM build AS build-k8s
+ARG KUBERNETES_VERSION
+ARG TAG
+WORKDIR /
+RUN git clone -b ${KUBERNETES_VERSION} --depth=1 https://github.com/kubernetes/kubernetes.git
+WORKDIR /kubernetes
+RUN make KUBE_GIT_VERSION=${TAG} WHAT='cmd/kube-apiserver cmd/kube-controller-manager cmd/kube-proxy cmd/kube-scheduler cmd/kubeadm cmd/kubectl cmd/kubelet vendor/k8s.io/apiextensions-apiserver'
+
+FROM registry.access.redhat.com/ubi7/ubi-minimal:latest AS kubernetes
+RUN microdnf update -y           && \
+    microdnf install -y iptables && \
+    rm -rf /var/cache/yum
+COPY --from=build-k8s \
+    /kubernetes/_output/bin/ \
+    /usr/local/bin/
+
 # rke-runtime image
 # This image includes any host level programs that we might need. All binaries
 # must be placed in bin/ of the file image and subdirectories of bin/ will be flattened during installation.
 # This means bin/foo/bar will become bin/bar when rke2 installs this to the host
-FROM rancher/kubernetes:${KUBERNETES_VERSION} AS k8s
 FROM rancher/k3s:v1.18.4-k3s1 AS k3s
 FROM rancher/containerd:v1.3.6-k3s2 AS containerd 
 
-FROM scratch AS release
-COPY --from=k8s \
-    /usr/local/bin/kubectl \
-    /usr/local/bin/kubelet \
-    /bin/
+FROM scratch AS runtime
 COPY --from=k3s \
     /bin/socat \
     /bin/runc \
@@ -51,4 +80,8 @@ COPY --from=containerd \
     /usr/local/bin/ctr \
     /usr/local/bin/containerd-shim-runc-v1 \
     /bin/
+COPY --from=kubernetes \
+    /usr/local/bin/kubectl \
+    /usr/local/bin/kubelet \
+    /bin/
 COPY ./build/static/charts /charts
diff --git a/Dockerfile.k8s b/Dockerfile.k8s
diff --git a/Makefile b/Makefile
@@ -23,24 +23,29 @@ dapper-ci: .ci                           ## Used by Drone CI, does the same as "
 build:                                   ## Build using host go tools
 	./scripts/build
 
-.PHONY: build-airgap
-build-airgap: | build image k8s-image build/images/airgap.tar	## Build all images for an airgapped installation
-
 .PHONY: build-debug
 build-debug:                             ## Debug build using host go tools
-	./scripts/build-debug
+	GODEBUG=y ./scripts/build
+
+.PHONY: build-images
+build-images:                             ## Build all images and image tarballs (including airgap)
+	./scripts/build-images
 
-.PHONY: image
-image: download-charts					## Build final docker image for push
-	./scripts/image
+.PHONY: build-image-kubernetes
+build-image-kubernetes:                               ## Build the kubernetes image
+	./scripts/build-image-kubernetes
 
-.PHONY: k8s-image
-k8s-image:                               ## Build final docker image for kubernetes
-	./scripts/k8s-image
+.PHONY: build-image-runtime
+build-image-runtime: build-charts					## Build the runtime image
+	./scripts/build-image-runtime
 
-.PHONY: image-publish
-image-publish: image
-	./scripts/image-publish
+.PHONY: publish-image-kubernetes
+publish-image-kubernetes: build-image-kubernetes
+	./scripts/publish-image-kubernetes
+
+.PHONY: publish-image-runtime
+publish-image-runtime: build-image-runtime
+	./scripts/publish-image-runtime
 
 .PHONY: validate
 validate:                                ## Run go fmt/vet
@@ -59,12 +64,9 @@ remote-debug-exit:              		 ## Kill dlv started with make remote-debug
 	./scripts/remote-debug-exit
 
 .PHONY: dev-shell-build
-dev-shell-build: build-airgap
+dev-shell-build: build
 	./scripts/dev-shell-build
 
-build/images/airgap.tar:
-	./scripts/airgap-images.sh
-
 .PHONY: clean-cache
 clean-cache:                             ## Clean up docker base caches used for development
 	./scripts/clean-cache
@@ -74,7 +76,7 @@ clean:                                   ## Clean up workspace
 	./scripts/clean
 
 .PHONY: dev-shell
-dev-shell: k8s-image download-charts image dev-shell-build              ## Launch a development shell to run test builds
+dev-shell: dev-shell-build              ## Launch a development shell to run test builds
 	./scripts/dev-shell
 
 .PHONY: dev-shell-enter
@@ -89,32 +91,32 @@ dev-peer: dev-shell-build              ## Launch a server peer to run test build
 dev-peer-enter:                         ## Enter the peer shell on another terminal
 	./scripts/dev-peer-enter
 
-.PHONY: k8s-image-publish
-k8s-image-publish: k8s-image
-	./scripts/k8s-image-publish
+.PHONY: publish-manifest-runtime
+publish-manifest-runtime: build-image-runtime					## Create and push the rke2-runtime manifest
+	./scripts/publish-manifest-runtime
 
-.PHONY: image-manifest
-image-manifest:							## mainfest rke2-runtime image
-	./scripts/image-manifest
+.PHONY: publish-manifest-kubernetes
+publish-manifest-kubernetes: build-image-kubernetes						## Create and push the kubernetes manifest
+	./scripts/publish-manifest-kubernetes
 
 .PHONY: dispatch
 dispatch:								## Send dispatch event to rke2-upgrade repo
 	./scripts/dispatch
 
-.PHONY: download-charts
-download-charts: 						## Download packaged helm charts
-	./scripts/download-charts
+.PHONY: build-charts
+build-charts: 						## Download packaged helm charts
+	./scripts/build-charts
 
 .PHONY: package
-package: | download-charts package-airgap package-bundle ## Package the rke2 binary
+package: build 						## Package the rke2 binary
 	./scripts/package
 
-.PHONY: package-airgap
-package-airgap: build/images/airgap.tar		## Package docker images for airgap environment
-	./scripts/package-airgap
+.PHONY: package-images
+package-images: build-images		## Package docker images for airgap environment
+	./scripts/package-images
 
 .PHONY: package-bundle
-package-bundle: 						## Package the tarball bundle
+package-bundle: build						## Package the tarball bundle
 	./scripts/package-bundle
 
 ./.dapper:

diff --git a/pkg/images/images.go b/pkg/images/images.go
@@ -47,7 +47,7 @@ func New(repo string) Images {
 		KubeControllManager: override(override("rancher", repo)+"/kubernetes:"+KubernetesVersion, controllerManager),
 		KubeScheduler:       override(override("rancher", repo)+"/kubernetes:"+KubernetesVersion, scheduler),
 		Pause:               override(override("k8s.gcr.io", repo)+"/pause:"+PauseVersion, pause),
-		Runtime:             override(override("rancher", repo)+"/rke2-runtime:"+version.Version, runtime),
+		Runtime:             override(override("rancher", repo)+"/rke2-runtime:"+strings.ReplaceAll(version.Version, "+", "-"), runtime),
 		ETCD:                override(override("rancher", repo)+"/etcd:"+EtcdVersion, etcd),
 	}
 }

diff --git a/scripts/airgap-images.sh b/scripts/airgap-images.sh
diff --git a/scripts/build b/scripts/build
@@ -1,38 +1,10 @@
-#!/bin/bash
-set -e -x
+#!/usr/bin/env bash
+set -ex
 
 cd $(dirname $0)/..
 
-. ./scripts/version.sh
-
-if [ -z "${GODEBUG}" ]; then
-    EXTRA_LDFLAGS="${EXTRA_LDFLAGS} -s -w"
-	DEBUG_GO_GCFLAGS=""
-    DEBUG_TAGS=""
-else
-    DEBUG_GO_GCFLAGS='-gcflags=all=-N -l'
-fi
-
-REVISION=$(git rev-parse HEAD)$(if ! git diff --no-ext-diff --quiet --exit-code; then echo .dirty; fi)
-RELEASE=${PROG}.${GOOS}-${GOARCH}
-
-
-BUILDTAGS=" netgo osusergo"
-GO_BUILDTAGS=${GO_BUILDTAGS}${BUILDTAGS}${DEBUG_TAGS}
-
-VERSION_FLAGS="
-        -X ${RKE2_PKG}/pkg/images.KubernetesVersion=${KUBERNETES_VERSION}
-        -X ${K3S_PKG}/pkg/version.Program=${PROG}
-        -X ${K3S_PKG}/pkg/version.Version=${VERSION}
-        -X ${K3S_PKG}/pkg/version.GitCommit=${REVISION}
-"
-STATIC_FLAGS='-extldflags "-static"'
-
-GO_LDFLAGS="${STATIC_FLAGS} ${EXTRA_LDFLAGS}"
-echo ${DEBUG_GO_GCFLAGS}
-go build \
-         -tags "${GO_BUILDTAGS}" \
-         ${GO_GCFLAGS} ${GO_BUILD_FLAGS} \
-         -o bin/${PROG} \
-         -ldflags "${GO_LDFLAGS} ${VERSION_FLAGS}" \
-         ${GO_TAGS}
+source ./scripts/version.sh
+mkdir -p build/images
+./scripts/build-binary
+./scripts/build-charts
+./scripts/build-images