From 7650794da9e07be25a44015d91f7606b1a00919c Mon Sep 17 00:00:00 2001 From: Kfir Toledo Date: Thu, 9 May 2024 10:05:53 +0300 Subject: [PATCH] Create support for arm64 architecture for docker images. (#564) Create support for arm64 architecture for docker images: 1. Update the GitHub action. 2. Update the makefile to compile images according to the architecture 3. Update the docker file to use the correct binary file. 4. Update the push image command to use the --platform flag for multi-architecture Docker image compilation. 5. Update the iperf3 demo to support arm64 architecture. Signed-off-by: Kfir Toledo --- .github/workflows/release.yml | 17 ++++---- Makefile | 41 +++++++++---------- cmd/cl-controlplane/Dockerfile | 5 ++- cmd/cl-dataplane/Dockerfile | 5 ++- cmd/cl-go-dataplane/Dockerfile | 5 ++- cmd/cl-operator/Dockerfile | 5 ++- demos/iperf3/cloud/test.py | 4 +- .../iperf3-client/iperf3-client.yaml | 4 +- .../iperf3-client/iperf3-client2.yaml | 4 +- .../manifests/iperf3-server/iperf3.yaml | 4 +- demos/utils/cloud.py | 8 +++- 11 files changed, 59 insertions(+), 43 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e6f0ed40..f5ac35df 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,8 +14,6 @@ jobs: steps: - name: checkout uses: actions/checkout@v4 - with: - fetch-tags: true - name: Set up Go uses: actions/setup-go@v5 with: @@ -26,12 +24,15 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build images - run: make docker-build - - name: Tag and push ClusterLink images with tag 'latest' - run: make push-image - - name: Tag and push ClusterLink images latest with tag ${{ github.ref_name }} - run: make push-image IMAGE_VERSION=${{ github.ref_name }} + - name: Build binaries for amd64 + run: GOARCH=amd64 make build + - name: Build binaries for arm64 + run: GOARCH=arm64 make build + - name: Tag and push ClusterLink images with tag 'latest' and ${{ github.ref_name }} + run: | + docker buildx create --use --driver docker-container + PLATFORMS=linux/amd64,linux/arm64 make push-image + PLATFORMS=linux/amd64,linux/arm64 make push-image IMAGE_VERSION=${{ github.ref_name }} - name: Build and compress binaries run: | for pair in "linux:amd64" "linux:arm64" "darwin:amd64" "darwin:arm64"; do diff --git a/Makefile b/Makefile index 8af44d47..f48eecdf 100755 --- a/Makefile +++ b/Makefile @@ -3,7 +3,8 @@ SHELL=/bin/bash IMAGE_VERSION ?= latest IMAGE_ORG ?= clusterlink-net IMAGE_BASE ?= ghcr.io/$(IMAGE_ORG) - +PLATFORMS ?= linux/amd64 +GOARCH ?=amd64 #----------------------------------------------------------------------------- # Target: clean #----------------------------------------------------------------------------- @@ -87,6 +88,7 @@ BIN_DIR := ./bin VERSION_FLAG := -X 'github.com/clusterlink-net/clusterlink/pkg/versioninfo.GitTag=$(shell git describe --tags --abbrev=0)' REVISION_FLAG := -X 'github.com/clusterlink-net/clusterlink/pkg/versioninfo.Revision=$(shell git rev-parse --short HEAD)' LD_FLAGS := -ldflags "$(VERSION_FLAG) $(REVISION_FLAG)" +export BUILDX_NO_DEFAULT_ATTESTATIONS := 1# Disable default attestations during Docker builds to prevent "unknown/unknown" image in ghcr. # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -118,29 +120,24 @@ cli-build: $(GO) build -o $(BIN_DIR)/clusterlink $(LD_FLAGS) ./cmd/clusterlink build: cli-build - $(GO) build -o $(BIN_DIR)/cl-controlplane $(LD_FLAGS) ./cmd/cl-controlplane - $(GO) build -o $(BIN_DIR)/cl-dataplane ./cmd/cl-dataplane - $(GO) build -o $(BIN_DIR)/cl-go-dataplane ./cmd/cl-go-dataplane - $(GO) build -o $(BIN_DIR)/cl-operator $(LD_FLAGS) ./cmd/cl-operator/main.go + GOARCH=$(GOARCH) $(GO) build -o $(BIN_DIR)/$(GOARCH)/cl-controlplane $(LD_FLAGS) ./cmd/cl-controlplane + GOARCH=$(GOARCH) $(GO) build -o $(BIN_DIR)/$(GOARCH)/cl-dataplane ./cmd/cl-dataplane + GOARCH=$(GOARCH) $(GO) build -o $(BIN_DIR)/$(GOARCH)/cl-go-dataplane ./cmd/cl-go-dataplane + GOARCH=$(GOARCH) $(GO) build -o $(BIN_DIR)/$(GOARCH)/cl-operator $(LD_FLAGS) ./cmd/cl-operator/main.go docker-build: build - docker build --progress=plain --rm --tag cl-controlplane -f ./cmd/cl-controlplane/Dockerfile . - docker build --progress=plain --rm --tag cl-dataplane -f ./cmd/cl-dataplane/Dockerfile . - docker build --progress=plain --rm --tag cl-go-dataplane -f ./cmd/cl-go-dataplane/Dockerfile . - docker build --progress=plain --rm --tag gwctl -f ./cmd/gwctl/Dockerfile . - docker build --progress=plain --rm --tag cl-operator -f ./cmd/cl-operator/Dockerfile . - -push-image: docker-build - docker tag cl-dataplane:latest $(IMAGE_BASE)/cl-dataplane:$(IMAGE_VERSION) - docker push $(IMAGE_BASE)/cl-dataplane:$(IMAGE_VERSION) - docker tag cl-controlplane:latest $(IMAGE_BASE)/cl-controlplane:$(IMAGE_VERSION) - docker push $(IMAGE_BASE)/cl-controlplane:$(IMAGE_VERSION) - docker tag cl-go-dataplane:latest $(IMAGE_BASE)/cl-go-dataplane:$(IMAGE_VERSION) - docker push $(IMAGE_BASE)/cl-go-dataplane:$(IMAGE_VERSION) - docker tag gwctl:latest $(IMAGE_BASE)/gwctl:$(IMAGE_VERSION) - docker push $(IMAGE_BASE)/gwctl:$(IMAGE_VERSION) - docker tag cl-operator:latest $(IMAGE_BASE)/cl-operator:$(IMAGE_VERSION) - docker push $(IMAGE_BASE)/cl-operator:$(IMAGE_VERSION) + docker build --platform $(PLATFORMS) --progress=plain --rm --tag cl-controlplane -f ./cmd/cl-controlplane/Dockerfile . + docker build --platform $(PLATFORMS) --progress=plain --rm --tag cl-dataplane -f ./cmd/cl-dataplane/Dockerfile . + docker build --platform $(PLATFORMS) --progress=plain --rm --tag cl-go-dataplane -f ./cmd/cl-go-dataplane/Dockerfile . + docker build --platform $(PLATFORMS) --progress=plain --rm --tag gwctl -f ./cmd/gwctl/Dockerfile . + docker build --platform $(PLATFORMS) --progress=plain --rm --tag cl-operator -f ./cmd/cl-operator/Dockerfile . + +push-image: build + docker buildx build --platform $(PLATFORMS) --progress=plain --rm --tag $(IMAGE_BASE)/cl-controlplane:$(IMAGE_VERSION) --push -f ./cmd/cl-controlplane/Dockerfile . + docker buildx build --platform $(PLATFORMS) --progress=plain --rm --tag $(IMAGE_BASE)/cl-go-dataplane:$(IMAGE_VERSION) --push -f ./cmd/cl-go-dataplane/Dockerfile . + docker buildx build --platform $(PLATFORMS) --progress=plain --rm --tag $(IMAGE_BASE)/cl-dataplane:$(IMAGE_VERSION) --push -f ./cmd/cl-dataplane/Dockerfile . + docker buildx build --platform $(PLATFORMS) --progress=plain --rm --tag $(IMAGE_BASE)/cl-operator:$(IMAGE_VERSION) --push -f ./cmd/cl-operator/Dockerfile . + docker buildx build --platform $(PLATFORMS) --progress=plain --rm --tag $(IMAGE_BASE)/gwctl:$(IMAGE_VERSION) --push -f ./cmd/gwctl/Dockerfile . install: mkdir -p ~/.local/bin diff --git a/cmd/cl-controlplane/Dockerfile b/cmd/cl-controlplane/Dockerfile index 8237c0d9..04b36ae7 100644 --- a/cmd/cl-controlplane/Dockerfile +++ b/cmd/cl-controlplane/Dockerfile @@ -1,8 +1,11 @@ FROM alpine:3.19 +# Populated during the build process, for example, with 'arm64' or 'amd64'. +ARG TARGETARCH + # Copy binary RUN mkdir -p /usr/local/bin -COPY ./bin/cl-controlplane /usr/local/bin/cl-controlplane +COPY ./bin/$TARGETARCH/cl-controlplane /usr/local/bin/cl-controlplane # Create directory for private keys RUN mkdir -p /etc/ssl/private diff --git a/cmd/cl-dataplane/Dockerfile b/cmd/cl-dataplane/Dockerfile index 68b2ee23..3801c08b 100644 --- a/cmd/cl-dataplane/Dockerfile +++ b/cmd/cl-dataplane/Dockerfile @@ -1,8 +1,11 @@ FROM envoyproxy/envoy:v1.30.1 +# Populated during the build process, for example, with 'arm64' or 'amd64'. +ARG TARGETARCH + # Copy binary RUN mkdir -p /usr/local/bin -COPY ./bin/cl-dataplane /usr/local/bin/cl-dataplane +COPY ./bin/$TARGETARCH/cl-dataplane /usr/local/bin/cl-dataplane # Create directory for private keys RUN mkdir -p /etc/ssl/private diff --git a/cmd/cl-go-dataplane/Dockerfile b/cmd/cl-go-dataplane/Dockerfile index ef8db147..c7934806 100644 --- a/cmd/cl-go-dataplane/Dockerfile +++ b/cmd/cl-go-dataplane/Dockerfile @@ -1,8 +1,11 @@ FROM alpine:3.19 +# Populated during the build process, for example, with 'arm64' or 'amd64'. +ARG TARGETARCH + # Copy binary RUN mkdir -p /usr/local/bin -COPY ./bin/cl-go-dataplane /usr/local/bin/cl-go-dataplane +COPY ./bin/$TARGETARCH/cl-go-dataplane /usr/local/bin/cl-go-dataplane # Create directory for private keys RUN mkdir -p /etc/ssl/private diff --git a/cmd/cl-operator/Dockerfile b/cmd/cl-operator/Dockerfile index 1cddaee3..187333e7 100644 --- a/cmd/cl-operator/Dockerfile +++ b/cmd/cl-operator/Dockerfile @@ -1,8 +1,11 @@ FROM alpine:3.14 + + FROM gcr.io/distroless/static:nonroot WORKDIR / -COPY ./bin/cl-operator cl-operator +ARG TARGETARCH +COPY ./bin/$TARGETARCH/cl-operator cl-operator USER 65532:65532 ENTRYPOINT ["/cl-operator"] diff --git a/demos/iperf3/cloud/test.py b/demos/iperf3/cloud/test.py index 8d0915c3..1687bd15 100755 --- a/demos/iperf3/cloud/test.py +++ b/demos/iperf3/cloud/test.py @@ -23,11 +23,11 @@ from demos.utils.cloud import Cluster from demos.iperf3.test import iperf3Test -cl1gcp = Cluster(name="peer1", zone = "us-west1-b", platform = "gcp") +cl1gcp = Cluster(name="peer1", zone = "us-central1-b", platform = "gcp") cl1aws = Cluster(name="peer1", zone = "us-west-2", platform = "aws") cl1ibm = Cluster(name="peer1", zone = "dal10", platform = "ibm") -cl2gcp = Cluster(name="peer2", zone = "us-west1-b", platform = "gcp") +cl2gcp = Cluster(name="peer2", zone = "us-central1-b", platform = "gcp") cl2aws = Cluster(name="peer2", zone = "us-west-1", platform = "aws") cl2ibm = Cluster(name="peer2", zone = "dal10", platform = "ibm") diff --git a/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client.yaml b/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client.yaml index 89d21f54..3eefe58d 100755 --- a/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client.yaml +++ b/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client.yaml @@ -1,5 +1,5 @@ ################################################################ -#Name: iperf3_client +#Name: iperf3_client #Desc: YAML file for creating iperf3 client to send test traffic. ################################################################ apiVersion: apps/v1 @@ -24,7 +24,7 @@ spec: containers: - name: iperf3-client #image: networkstatic/iperf3 - image: mlabbe/iperf3 + image: taoyou/iperf3-alpine imagePullPolicy: IfNotPresent command: ['/bin/sh', '-c', 'sleep infinity'] # To benchmark manually: kubectl exec iperf3-client-jlfxq -- /bin/sh -c 'iperf3 -c iperf3-server' diff --git a/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client2.yaml b/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client2.yaml index dc641aa9..9aff32c5 100755 --- a/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client2.yaml +++ b/demos/iperf3/testdata/manifests/iperf3-client/iperf3-client2.yaml @@ -1,5 +1,5 @@ ################################################################ -#Name: iperf3_client +#Name: iperf3_client #Desc: YAML file for creating iperf3 client to send test traffic. ################################################################ apiVersion: apps/v1 @@ -24,7 +24,7 @@ spec: containers: - name: iperf3-client2 #image: networkstatic/iperf3 - image: mlabbe/iperf3 + image: taoyou/iperf3-alpine imagePullPolicy: IfNotPresent command: ['/bin/sh', '-c', 'sleep infinity'] # To benchmark manually: kubectl exec iperf3-client2-jlfxq -- /bin/sh -c 'iperf3 -c iperf3-server' diff --git a/demos/iperf3/testdata/manifests/iperf3-server/iperf3.yaml b/demos/iperf3/testdata/manifests/iperf3-server/iperf3.yaml index 9328774b..e78738b9 100755 --- a/demos/iperf3/testdata/manifests/iperf3-server/iperf3.yaml +++ b/demos/iperf3/testdata/manifests/iperf3-server/iperf3.yaml @@ -1,5 +1,5 @@ ################################################################ -#Name: iperf3 +#Name: iperf3 #Desc: YAML file for creating iperf3 server for testing. ################################################################ apiVersion: apps/v1 @@ -44,7 +44,7 @@ spec: containers: - name: iperf3-server #image: networkstatic/iperf3 - image: mlabbe/iperf3 + image: taoyou/iperf3-alpine imagePullPolicy: IfNotPresent args: ['-s', '-p', '5000'] # ports: diff --git a/demos/utils/cloud.py b/demos/utils/cloud.py index ac91a953..381acf1c 100644 --- a/demos/utils/cloud.py +++ b/demos/utils/cloud.py @@ -36,7 +36,11 @@ def createCluster(self, runBg): print(f"create {self.name} cluster , zone {self.zone} , platform {self.platform}") bgFlag = " &" if runBg else "" if self.platform == "gcp": - flags = " --machine-type n2-standard-4" if self.machineType=="large" else "" #e2-medium + machine_map = { + "large": "--machine-type n2-standard-4", + "arm64": "--machine-type t2a-standard-1" + } + flags = machine_map.get(self.machineType, "") # default e2-medium cmd=f"gcloud container clusters create {self.name} --zone {self.zone} --num-nodes 1 --tags tcpall {flags} {bgFlag}" print(cmd) os.system(cmd) @@ -68,6 +72,8 @@ def startCluster(self, testOutputFolder, logLevel="info",dataplane="envoy"): self.useCluster() super().set_kube_config() self.create_peer_cert(self.name,testOutputFolder) + if self.platform =="gcp" and self.machineType == "arm64": # Allow to schedule pods in arm64 on gcp + runcmd("kubectl taint nodes ` kubectl get nodes -o wide | awk '{if (NR>1) print $1}'` kubernetes.io/arch=arm64:NoSchedule-") self.deploy_peer(self.name, testOutputFolder, logLevel, dataplane) self.waitToLoadBalancer() self.nodeIP = getNodeIP(num=1)