build hybrid image

Signed-off-by: Lyndon-Li <[email protected]>

Dockerfile-Windows

@@ -38,8 +38,6 @@ ENV CGO_ENABLED=0 \
 
 WORKDIR /go/src/github.com/vmware-tanzu/velero
 
-RUN apt-get update && apt-get install -y jq
-
 COPY . /go/src/github.com/vmware-tanzu/velero
 
 RUN mkdir -p /output/usr/bin && \
@@ -52,7 +50,8 @@ RUN mkdir -p /output/usr/bin && \
 
 # Velero image packing section
 FROM mcr.microsoft.com/windows/nanoserver:${OS_VERSION}
-ENTRYPOINT [ "velero.exe" ]
+ARG BIN
+ENTRYPOINT [ "${BIN}.exe" ]
 COPY --from=velero-builder /output /
 
 USER ContainerUser

Makefile

@@ -101,30 +101,28 @@ comma=,
 RESTIC_VERSION ?= 0.15.0
 
 CLI_PLATFORMS ?= linux-amd64 linux-arm linux-arm64 darwin-amd64 darwin-arm64 windows-amd64 linux-ppc64le
-BUILDX_PUSH ?= false
-BUILDX_BUILD_OS ?= linux
-BUILDX_BUILD_ARCH ?= amd64
-BUILDX_TAG_GCR ?= false
-BUILDX_WINDOWS_VERSION ?= ltsc2022
+BUILD_OUTPUT_TYPE ?= docker
+BUILD_OS ?= linux
+BUILD_ARCH ?= amd64
+BUILD_TAG_GCR ?= false
+BUILD_WINDOWS_VERSION ?= ltsc2022
 
-ifneq ($(BUILDX_PUSH), true)
+ifeq ($(BUILD_OUTPUT_TYPE), docker)
 	ALL_OS = linux
 	ALL_ARCH.linux = $(word 2, $(subst -, ,$(shell go env GOOS)-$(shell go env GOARCH)))
-	BUILDX_OUTPUT_TYPE = docker
 else
-	ALL_OS = $(subst $(comma), ,$(BUILDX_BUILD_OS))
-	ALL_ARCH.linux = $(subst $(comma), ,$(BUILDX_BUILD_ARCH))
-	BUILDX_OUTPUT_TYPE = registry
+	ALL_OS = $(subst $(comma), ,$(BUILD_OS))
+	ALL_ARCH.linux = $(subst $(comma), ,$(BUILD_ARCH))
 endif
 
 ALL_ARCH.windows = $(if $(filter windows,$(ALL_OS)),amd64,)
-ALL_OSVERSIONS.windows = $(if $(filter windows,$(ALL_OS)),$(BUILDX_WINDOWS_VERSION),)
+ALL_OSVERSIONS.windows = $(if $(filter windows,$(ALL_OS)),$(BUILD_WINDOWS_VERSION),)
 ALL_OS_ARCH.linux =  $(foreach os, $(filter linux,$(ALL_OS)), $(foreach arch, ${ALL_ARCH.linux}, ${os}-$(arch)))
 ALL_OS_ARCH.windows = $(foreach os, $(filter windows,$(ALL_OS)), $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, ${os}-${osversion}-${arch})))
 ALL_OS_ARCH = $(ALL_OS_ARCH.linux)$(ALL_OS_ARCH.windows)
 
 ALL_IMAGE_TAGS = $(IMAGE_TAGS)
-ifeq ($(BUILDX_TAG_GCR), true)
+ifeq ($(BUILD_TAG_GCR), true)
 	ALL_IMAGE_TAGS += $(GCR_IMAGE_TAGS)
 endif
 
@@ -225,11 +223,13 @@ endif
 	-docker buildx rm velero-builder || true
 	@docker buildx create --use --name=velero-builder
 
+	@mkdir -p _output
+
 	@for osarch in $(ALL_OS_ARCH); do \
 		$(MAKE) container-$${osarch}; \
 	done
 
-ifeq ($(BUILDX_PUSH), true)
+ifeq ($(BUILD_OUTPUT_TYPE), registry)
 	@for tag in $(ALL_IMAGE_TAGS); do \
 		IMAGE_TAG=$${tag} $(MAKE) push-manifest; \
 	done
@@ -242,7 +242,7 @@ container-linux:
 	@echo "building container: $(IMAGE):$(VERSION)-linux-$(BUILDX_ARCH)"
 
 	@docker buildx build --pull \
-	--output=type=$(BUILDX_OUTPUT_TYPE) \
+	--output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-linux-$(BUILDX_ARCH).tar,)" \
 	--platform="linux/$(BUILDX_ARCH)" \
 	$(addprefix -t , $(addsuffix "-linux-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \
 	--build-arg=GOPROXY=$(GOPROXY) \
@@ -266,7 +266,7 @@ container-windows:
 	@echo "building container: $(IMAGE):$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)"
 
 	@docker buildx build --pull \
-	--output=type=$(BUILDX_OUTPUT_TYPE) \
+	--output="type=$(BUILD_OUTPUT_TYPE)$(if $(findstring tar, $(BUILD_OUTPUT_TYPE)),$(comma)dest=_output/$(BIN)-$(VERSION)-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH).tar,)" \
 	--platform="windows/$(BUILDX_ARCH)" \
 	$(addprefix -t , $(addsuffix "-windows-$(BUILDX_OSVERSION)-$(BUILDX_ARCH)",$(ALL_IMAGE_TAGS))) \
 	--build-arg=GOPROXY=$(GOPROXY) \
@@ -296,10 +296,10 @@ push-manifest:
 		done; \
 	done
 
-	@echo "pushing mainifest $(IMAGE_TAG)"
+	@echo "pushing manifest $(IMAGE_TAG)"
 	@docker manifest push --purge $(IMAGE_TAG)
 
-	@echo "pushed mainifest $(IMAGE_TAG):"
+	@echo "pushed manifest $(IMAGE_TAG):"
 	@docker manifest inspect $(IMAGE_TAG)
 
 SKIP_TESTS ?=

changelogs/unreleased/8476-Lyndon-Li

@@ -0,0 +1 @@
+Fix issue #8415, implement multi-arch build and Windows build

design/multiple-arch-build-with-windows.md

@@ -76,7 +76,7 @@ The built image could be listed by `docker image ls`.
 
 **Local build for linux-amd64 and windows-amd64 to tar**
 ```
-BUILDX_OUTPUT_TYPE=tar BUILD_OS=linux,windows make container
+BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows make container
 ```
 Under `_output` directory, below files are generated:  
 ```
@@ -86,7 +86,7 @@ velero-main-windows-ltsc2022-amd64.tar
 
 **Local build for linux-amd64, linux-arm64 and windows-amd64 to tar**
 ```
-BUILDX_OUTPUT_TYPE=tar BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
+BUILD_OUTPUT_TYPE=tar BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
 ```
 Under `_output` directory, below files are generated:  
 ```
@@ -98,7 +98,7 @@ velero-main-windows-ltsc2022-amd64.tar
 **Push build for linux-amd64 and windows-amd64**  
 Prerequisite: login to registry, e.g., through `docker login`  
 ```
-BUILDX_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows make container
+BUILD_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows make container
 ```
 Nothing is available locally, in the registry 3 tags are available:
 ```
@@ -110,7 +110,7 @@ velero/velero:main-linux-amd64
 **Push build for linux-amd64, linux-arm64 and windows-amd64**  
 Prerequisite: login to registry, e.g., through `docker login` 
 ```
-BUILDX_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
+BUILD_OUTPUT_TYPE=registry REGISTRY=<registry> BUILD_OS=linux,windows BUILD_ARCH=amd64,arm64 make container
 ```
 Nothing is available locally, in the registry 4 tags are available:
 ```

hack/docker-push.sh

@@ -63,7 +63,7 @@ fi
 if [[ -z $BRANCH && -z $TAG ]]; then
     echo "Test Velero container build without pushing, when Dockerfile is changed by PR."
     BRANCH="${GITHUB_BASE_REF}-container"
-    OUTPUT_TYPE="local,dest=."
+    OUTPUT_TYPE="tar"
 else
     OUTPUT_TYPE="registry"
 fi
@@ -88,8 +88,12 @@ else
     fi
 fi
 
-if [[ -z "$BUILDX_PLATFORMS" ]]; then
-    BUILDX_PLATFORMS="linux/amd64,linux/arm64"
+if [[ -z "$BUILD_OS" ]]; then
+    BUILD_OS="linux,windows"
+fi
+
+if [[ -z "$BUILD_ARCH" ]]; then
+    BUILD_ARCH="amd64,arm64"
 fi
 
 # Debugging info
@@ -98,13 +102,15 @@ echo "BRANCH: $BRANCH"
 echo "TAG: $TAG"
 echo "TAG_LATEST: $TAG_LATEST"
 echo "VERSION: $VERSION"
-echo "BUILDX_PLATFORMS: $BUILDX_PLATFORMS"
+echo "BUILD_OS: $BUILD_OS"
+echo "BUILD_ARCH: $BUILD_ARCH"
 
 echo "Building and pushing container images."
 
 
 VERSION="$VERSION" \
 TAG_LATEST="$TAG_LATEST" \
-BUILDX_PLATFORMS="$BUILDX_PLATFORMS" \
-BUILDX_OUTPUT_TYPE=$OUTPUT_TYPE \
-make all-containers
+BUILD_OS="$BUILD_OS" \
+BUILD_ARCH="$BUILD_ARCH" \
+BUILD_OUTPUT_TYPE=$OUTPUT_TYPE \
+make all-containers