ci: Improve cross-compile performance

Improve cross-compile performance using rust cross-compiler instead of buildx Signed-off-by: Seunguk Shin <[email protected]> Reviewed-by: Nick Connolly <[email protected]>
confidential-containers · Dec 20, 2024 · d1b994d · d1b994d
1 parent 36e52c6
commit d1b994d
Show file tree

Hide file tree

Showing 10 changed files with 107 additions and 34 deletions.
diff --git a/.github/workflows/build-as-image.yml b/.github/workflows/build-as-image.yml
@@ -35,14 +35,17 @@ jobs:
         # add instance and verifier flag to target
         - target_arch: x86_64
           target_platform: linux/amd64
+          build_platform: linux/amd64
           instance: ubuntu-latest
           verifier: all-verifier
         - target_arch: s390x
           target_platform: linux/s390x
+          build_platform: linux/s390x
           instance: s390x
           verifier: se-verifier
         - target_arch: aarch64
           target_platform: linux/arm64
+          build_platform: linux/amd64
           instance: ubuntu-latest
           verifier: cca-verifier
     runs-on: ${{ matrix.instance }}
@@ -66,6 +69,7 @@ jobs:
         commit_sha=${{ github.sha }}
         docker buildx build --platform "${{ matrix.target_platform }}" \
           -f "${{ matrix.docker_file }}" ${{ inputs.build_option }} \
+          --build-arg BUILDPLATFORM="${{ matrix.build_platform }}" \
           --build-arg ARCH="${{ matrix.target_arch }}" \
           --build-arg VERIFIER="${{ matrix.verifier }}" \
           -t "ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-${{ matrix.target_arch }}" \

diff --git a/.github/workflows/build-kbs-image.yml b/.github/workflows/build-kbs-image.yml
@@ -48,12 +48,15 @@ jobs:
           # add instance flag to target
           - target_arch: x86_64
             target_platform: linux/amd64
+            build_platform: linux/amd64
             instance: ubuntu-latest
           - target_arch: s390x
             target_platform: linux/s390x
+            build_platform: linux/s390x
             instance: s390x
           - target_arch: aarch64
             target_platform: linux/arm64
+            build_platform: linux/amd64
             instance: ubuntu-latest
 
     runs-on: ${{ matrix.instance }}
@@ -79,4 +82,5 @@ jobs:
           -f "${{ matrix.docker_file }}" ${{ inputs.build_option }} \
           -t "ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:${commit_sha}-${{ matrix.target_arch }}" \
           -t "ghcr.io/confidential-containers/staged-images/${{ matrix.tag }}:latest-${{ matrix.target_arch }}" \
+          --build-arg BUILDPLATFORM="${{ matrix.build_platform }}" \
           --build-arg ARCH="${{ matrix.target_arch }}" .
diff --git a/.github/workflows/push-kbs-client-to-ghcr.yml b/.github/workflows/push-kbs-client-to-ghcr.yml
@@ -14,13 +14,6 @@ jobs:
           - x86_64
           - s390x
           - aarch64
-        include:
-          - arch: x86_64
-            platform: linux/amd64
-          - arch: s390x
-            platform: linux/s390x
-          - arch: aarch64
-            platform: linux/arm64
     runs-on: ${{ matrix.arch == 's390x' && 's390x' || 'ubuntu-22.04' }}
     permissions:
       contents: read
@@ -42,8 +35,7 @@ jobs:
 
     - name: Build a statically linked kbs-client for ${{ matrix.arch }} linux
       run: |
-        docker buildx build --platform "${{ matrix.platform }}" \
-          -f kbs/docker/kbs-client/Dockerfile \
+        docker buildx build -f kbs/docker/kbs-client/Dockerfile \
           --build-arg ARCH="${{ matrix.arch }}" --output ./ .
 
     - name: Push to ghcr.io

diff --git a/attestation-service/docker/as-grpc/Dockerfile b/attestation-service/docker/as-grpc/Dockerfile
@@ -2,7 +2,8 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM rust:latest AS builder
+FROM --platform=$BUILDPLATFORM rust:latest AS builder
+ARG BUILDPLATFORM=linux/amd64
 ARG ARCH=x86_64
 ARG VERIFIER=all-verifier
 
@@ -18,7 +19,16 @@ RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/
     apt-get update && apt-get install -y libsgx-dcap-quote-verify-dev; fi
 
 # Build and Install gRPC attestation-service
-RUN cargo install --path attestation-service --bin grpc-as --features grpc-bin,${VERIFIER} --locked
+RUN if [ "$(uname -m)" != "${ARCH}" ]; then \
+    export GCC_PACKAGE="gcc-${ARCH}-linux-gnu"; \
+    export GCC_COMPILER="${ARCH}-linux-gnu-gcc"; \
+    export RUSTC_TARGET="${ARCH}-unknown-linux-gnu"; \
+    export TARGET_FLAG="--target ${RUSTC_TARGET}"; \
+    export RUSTFLAGS_ARGS=" -C linker=${GCC_COMPILER}"; \
+    export RUSTFLAGS="${RUSTFLAGS_ARGS}"; \
+    apt-get install -y ${GCC_PACKAGE}; \
+    rustup target add ${RUSTC_TARGET}; fi; \
+    cargo install --path attestation-service --bin grpc-as --features grpc-bin,${VERIFIER} --locked ${TARGET_FLAG}
 
 
 FROM ubuntu:22.04

diff --git a/attestation-service/docker/as-restful/Dockerfile b/attestation-service/docker/as-restful/Dockerfile
@@ -2,7 +2,8 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM rust:latest AS builder
+FROM --platform=$BUILDPLATFORM rust:latest AS builder
+ARG BUILDPLATFORM=linux/amd64
 ARG ARCH=x86_64
 ARG VERIFIER=all-verifier
 
@@ -18,7 +19,16 @@ RUN if [ "${ARCH}" = "x86_64" ]; then curl -L https://download.01.org/intel-sgx/
     apt-get update && apt-get install -y libsgx-dcap-quote-verify-dev; fi
 
 # Build and Install RESTful attestation-service
-RUN cargo install --path attestation-service --bin restful-as --features restful-bin,${VERIFIER} --locked
+RUN if [ "$(uname -m)" != "${ARCH}" ]; then \
+    export GCC_PACKAGE="gcc-${ARCH}-linux-gnu"; \
+    export GCC_COMPILER="${ARCH}-linux-gnu-gcc"; \
+    export RUSTC_TARGET="${ARCH}-unknown-linux-gnu"; \
+    export TARGET_FLAG="--target ${RUSTC_TARGET}"; \
+    export RUSTFLAGS_ARGS=" -C linker=${GCC_COMPILER}"; \
+    export RUSTFLAGS="${RUSTFLAGS_ARGS}"; \
+    apt-get install -y ${GCC_PACKAGE}; \
+    rustup target add ${RUSTC_TARGET}; fi; \
+    cargo install --path attestation-service --bin restful-as --features restful-bin,${VERIFIER} --locked ${TARGET_FLAG}
 
 FROM ubuntu:22.04
 ARG ARCH=x86_64

diff --git a/kbs/Makefile b/kbs/Makefile
@@ -1,12 +1,35 @@
 AS_TYPE ?= coco-as
 ALIYUN ?= false
 
-ARCH := $(shell uname -m)
+BUILD_ARCH := $(shell uname -m)
+ARCH ?= $(shell uname -m)
 # Check if ARCH is supported, otehrwise return error
 ifeq ($(filter $(ARCH),x86_64 s390x aarch64),)
 	$(error "Unsupported architecture: $(ARCH)")
 endif
 
+RELEASE_DIR := ../target/release
+TARGET_FLAG :=
+CARGO_ENV :=
+ifneq ($(BUILD_ARCH), $(ARCH))
+  ifneq (,$(wildcard /etc/debian_version))
+    GCC_PACKAGE := gcc-$(ARCH)-linux-gnu
+    GCC_COMPILER := $(ARCH)-linux-gnu-gcc
+    RUSTC_TARGET := $(ARCH)-unknown-linux-gnu
+    GCC_INSTALL := $(shell sudo apt-get install -y ${GCC_PACKAGE})
+    RUST_INSTALL := $(shell rustup target add ${RUSTC_TARGET})
+    RUSTFLAGS_ARGS := -C linker=$(GCC_COMPILER)
+    TARGET_FLAG := --target $(RUSTC_TARGET)
+    RELEASE_DIR := ../target/$(RUSTC_TARGET)/release
+    OS_ARCH := $(ARCH)
+    OS_ARCH := $(OS_ARCH:x86_64=amd64)
+    OS_ARCH := $(OS_ARCH:aarch64=arm64)
+    CARGO_ENV := OPENSSL_INCLUDE_DIR=/usr/include/$(ARCH)-linux-gnu OPENSSL_LIB_DIR=/usr/lib/$(ARCH)-linux-gnu RUSTFLAGS="$(RUSTFLAGS_ARGS)"
+  else
+    $(error ERROR: Cross-compiling is only tested on Debian-like OSes)
+  endif
+endif
+
 CLI_FEATURES ?=
 ATTESTER ?=
 FEATURES ?=
@@ -37,25 +60,25 @@ build: background-check-kbs
 
 .PHONY: background-check-kbs
 background-check-kbs:
-	cargo build -p kbs --locked --release --no-default-features --features $(FEATURES),$(AS_FEATURE)
+	$(CARGO_ENV) cargo build -p kbs --locked --release --no-default-features --features $(FEATURES),$(AS_FEATURE) $(TARGET_FLAG)
 
 .PHONY: passport-issuer-kbs
 passport-issuer-kbs:
-	cargo build -p kbs --locked --release --no-default-features --features $(AS_FEATURE),$(FEATURES)
+	$(CARGO_ENV) cargo build -p kbs --locked --release --no-default-features --features $(AS_FEATURE),$(FEATURES) $(TARGET_FLAG)
 	mv ../target/release/kbs ../target/release/issuer-kbs
 
 .PHONY: passport-resource-kbs
 passport-resource-kbs:
-	cargo build -p kbs --locked --release --no-default-features --features $(FEATURES),
+	$(CARGO_ENV) cargo build -p kbs --locked --release --no-default-features --features $(FEATURES), $(TARGET_FLAG)
 	mv ../target/release/kbs ../target/release/resource-kbs
 
 .PHONY: cli
 cli:
-	cargo build -p kbs-client --locked --release --no-default-features --features $(CLI_FEATURES)
+	$(CARGO_ENV) cargo build -p kbs-client --locked --release --no-default-features --features $(CLI_FEATURES) $(TARGET_FLAG)
 
 .PHONY: cli-static-linux
 cli-static-linux:
-	cargo build \
+	$(CARGO_ENV) cargo build \
       -p kbs-client \
       --target=$(ARCH)-unknown-linux-gnu \
       --config "target.$(ARCH)-unknown-linux-gnu.rustflags = '-C target-feature=+crt-static'" \
@@ -65,17 +88,17 @@ cli-static-linux:
       --features sample_only
 
 install-kbs:
-	install -D -m0755 ../target/release/kbs $(INSTALL_DESTDIR)
+	install -D -m0755 $(RELEASE_DIR)/kbs $(INSTALL_DESTDIR)
 
 install-issuer-kbs:
-	install -D -m0755 ../target/release/issuer-kbs $(INSTALL_DESTDIR)
-	install -D -m0755 ../target/release/kbs-client $(INSTALL_DESTDIR)
+	install -D -m0755 $(RELEASE_DIR)/issuer-kbs $(INSTALL_DESTDIR)
+	install -D -m0755 $(RELEASE_DIR)/kbs-client $(INSTALL_DESTDIR)
 
 install-resource-kbs:
-	install -D -m0755 ../target/release/resource-kbs $(INSTALL_DESTDIR)
+	install -D -m0755 $(RELEASE_DIR)/resource-kbs $(INSTALL_DESTDIR)
 
 install-cli:
-	install -D -m0755 ../target/release/kbs-client $(INSTALL_DESTDIR)
+	install -D -m0755 $(RELEASE_DIR)/kbs-client $(INSTALL_DESTDIR)
 
 uninstall:
 	rm -rf $(INSTALL_DESTDIR)/kbs $(INSTALL_DESTDIR)/kbs-client $(INSTALL_DESTDIR)/issuer-kbs $(INSTALL_DESTDIR)/resource-kbs

diff --git a/kbs/docker/Dockerfile b/kbs/docker/Dockerfile
@@ -1,4 +1,5 @@
-FROM rust:slim AS builder
+FROM --platform=$BUILDPLATFORM rust:slim AS builder
+ARG BUILDPLATFORM=linux/amd64
 ARG ARCH=x86_64
 ARG ALIYUN=false
 
@@ -9,7 +10,8 @@ RUN apt-get update && \
     curl \
     gpg \
     gnupg-agent \
-    git
+    git \
+    sudo
 
 RUN if [ "${ARCH}" = "x86_64" ]; then curl -fsSL https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | \
     gpg --dearmor --output /usr/share/keyrings/intel-sgx.gpg && \
@@ -36,8 +38,8 @@ RUN if [ "${ARCH}" = "x86_64" ]; then curl -fsSL https://download.01.org/intel-s
 WORKDIR /usr/src/kbs
 COPY . .
 
-RUN cd kbs && make AS_FEATURE=coco-as-builtin ALIYUN=${ALIYUN} && \
-    make install-kbs
+RUN cd kbs && make AS_FEATURE=coco-as-builtin ALIYUN=${ALIYUN} ARCH=${ARCH} && \
+    make ARCH=${ARCH} install-kbs
 
 FROM ubuntu:22.04
 ARG ARCH=x86_64

diff --git a/kbs/docker/coco-as-grpc/Dockerfile b/kbs/docker/coco-as-grpc/Dockerfile
@@ -1,15 +1,24 @@
-FROM rust:latest AS builder
+FROM --platform=$BUILDPLATFORM rust:latest AS builder
+ARG BUILDPLATFORM=linux/amd64
 ARG ARCH=x86_64
 ARG ALIYUN=false
 
 WORKDIR /usr/src/kbs
 COPY . .
 
-RUN apt-get update && apt install -y protobuf-compiler git
+RUN apt-get update && apt install -y protobuf-compiler git sudo
+
+ENV OS_ARCH=${ARCH}
+RUN if [ $(uname -m) != ${ARCH} ]; then \
+    OS_ARCH=$(echo $OS_ARCH | sed s/x86_64/amd64/); \
+    OS_ARCH=$(echo $OS_ARCH | sed s/aarch64/arm64/); \
+    dpkg --add-architecture ${OS_ARCH}; \
+    apt-get update; \
+    apt-get install -y libssl-dev:${OS_ARCH}; fi
 
 # Build and Install KBS
-RUN cd kbs && make AS_FEATURE=coco-as-grpc ALIYUN=${ALIYUN} && \
-    make install-kbs
+RUN cd kbs && make AS_FEATURE=coco-as-grpc ALIYUN=${ALIYUN} ARCH=${ARCH} && \
+    make ARCH=${ARCH} install-kbs
 
 FROM ubuntu:22.04
 

diff --git a/kbs/docker/kbs-client/Dockerfile b/kbs/docker/kbs-client/Dockerfile
@@ -6,6 +6,14 @@ COPY . .
 
 RUN apt-get update && apt install -y pkg-config libssl-dev git sudo
 
+ENV OS_ARCH=${ARCH}
+RUN if [ $(uname -m) != ${ARCH} ]; then \
+    OS_ARCH=$(echo $OS_ARCH | sed s/x86_64/amd64/); \
+    OS_ARCH=$(echo $OS_ARCH | sed s/aarch64/arm64/); \
+    dpkg --add-architecture ${OS_ARCH}; \
+    apt-get update; \
+    apt-get install -y libssl-dev:${OS_ARCH}; fi
+
 # Build KBS Client
 RUN cd kbs && make ARCH=${ARCH} cli-static-linux && \
     cp ../target/${ARCH}-unknown-linux-gnu/release/kbs-client /

diff --git a/rvps/docker/Dockerfile b/rvps/docker/Dockerfile
@@ -2,15 +2,26 @@
 # Licensed under the Apache License, Version 2.0, see LICENSE for details.
 # SPDX-License-Identifier: Apache-2.0
 
-FROM rust:latest AS builder
+FROM --platform=$BUILDPLATFORM rust:latest AS builder
+ARG BUILDPLATFORM=linux/amd64
+ARG ARCH=x86_64
 
 WORKDIR /usr/src/rvps
 
 COPY . .
 
 RUN apt-get update && apt-get install protobuf-compiler -y
 
-RUN cargo install --bin rvps --path rvps
+RUN if [ "$(uname -m)" != "${ARCH}" ]; then \
+    export GCC_PACKAGE="gcc-${ARCH}-linux-gnu"; \
+    export GCC_COMPILER="${ARCH}-linux-gnu-gcc"; \
+    export RUSTC_TARGET="${ARCH}-unknown-linux-gnu"; \
+    export TARGET_FLAG="--target ${RUSTC_TARGET}"; \
+    export RUSTFLAGS_ARGS=" -C linker=${GCC_COMPILER}"; \
+    export RUSTFLAGS="${RUSTFLAGS_ARGS}"; \
+    apt-get install -y ${GCC_PACKAGE}; \
+    rustup target add ${RUSTC_TARGET}; fi; \
+    cargo install --bin rvps --path rvps ${TARGET_FLAG}
 
 FROM debian