From ffdbfd30453354e6c071bef66fd3a33d657cdf16 Mon Sep 17 00:00:00 2001 From: andream16 Date: Wed, 11 Sep 2024 15:17:06 +0100 Subject: [PATCH] introduce multi-platform builds #334 This commit extends the base Dockerfile so that it can copy a binary from an arbitrary path and copy it into a target one inside the filesystem of the container image. The scripts building the component binaries and containers have also been extended with environment variables to help them build the binaries for the expected architecture. The Makefile has acquired also the same environment variable which are set by default in the host's architecture. In the case of the `dev-deploy` target, the architecture is set by default to `linux/amd64` since containers are usually executed on linux/amd64 systems or on linux/amd64 VMs running the Docker daemon (Windows, some Macs) or on VMs running linux/arm64 which supports amd64 binaries out of the box. --- Makefile | 39 ++++++++++++++----- components/producers/aggregator/Makefile | 8 ++-- .../eslint-wrapper/Dockerfile | 5 ++- .../typescript-eslint/eslint-wrapper/Makefile | 5 ++- containers/Dockerfile.draconctl | 6 ++- scripts/build_component_binary.sh | 9 ++++- scripts/build_component_container.sh | 39 ++++++++++++------- 7 files changed, 78 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 6d948be06..c9c2de33f 100644 --- a/Makefile +++ b/Makefile @@ -46,22 +46,31 @@ export .PHONY: components component-binaries cmd/draconctl/bin protos build publish-component-containers publish-containers draconctl-image draconctl-image-publish clean-protos clean $(component_binaries): - CGO_ENABLED=0 ./scripts/build_component_binary.sh $@ + ./scripts/build_component_binary.sh $@ component-binaries: $(component_binaries) $(component_containers): %/docker: %/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) ./scripts/build_component_container.sh $@ components: $(component_containers) cmd/draconctl/bin: - CGO_ENABLED=0 go build -o bin/cmd/draconctl cmd/draconctl/main.go + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o bin/cmd/$(GOOS)/$(GOARCH)/draconctl cmd/draconctl/main.go draconctl-image: cmd/draconctl/bin + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(DOCKER) build -t "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" \ + --build-arg GOOS=$(GOOS) \ + --build-arg GOARCH=$(GOARCH) \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f containers/Dockerfile.draconctl . + -f containers/Dockerfile.draconctl . \ + --platform "$(GOOS)/$(GOARCH)" draconctl-image-publish: draconctl-image $(DOCKER) push "${CONTAINER_REPO}/draconctl:${DRACON_VERSION}" @@ -266,18 +275,30 @@ install-oss-components: --values ./deploy/deduplication-db-migrations/values/dev.yaml @echo "Done! Bumped version to $(DRACON_VERSION)" -dev-build-oss-components: cmd/draconctl/bin +dev-build-oss-components: @echo "Building open-source components for local dracon instance..." + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) + $(eval TMP_DIR:=tmp) + @mkdir $(TMP_DIR) + $(MAKE) cmd/draconctl/bin $(MAKE) -j 16 publish-component-containers CONTAINER_REPO=$(CONTAINER_REPO) - @./bin/cmd/draconctl components package \ - --version $(DRACON_VERSION) \ - --chart-version $(DRACON_VERSION) \ - --name $(DRACON_OSS_COMPONENTS_NAME) \ - ./components + @docker run \ + --platform $(GOOS)/$(GOARCH) \ + -v ./components:/components \ + -v ./tmp:/tmp \ + $(CONTAINER_REPO)/draconctl:$(DRACON_VERSION) components package \ + --version $(DRACON_VERSION) \ + --chart-version $(DRACON_VERSION) \ + --name $(DRACON_OSS_COMPONENTS_NAME) \ + ./components + @rm -r $(TMP_DIR) dev-dracon: + $(eval GOOS:=linux) + $(eval GOARCH:=amd64) $(eval CONTAINER_REPO:=localhost:5000/ocurity/dracon) $(eval DRACON_OSS_COMPONENTS_PACKAGE_URL:=./$(DRACON_OSS_COMPONENTS_NAME)-$(DRACON_VERSION).tgz) $(eval IN_CLUSTER_CONTAINER_REPO:=kind-registry:5000/ocurity/dracon) diff --git a/components/producers/aggregator/Makefile b/components/producers/aggregator/Makefile index 8009b1350..2e3bcc0c8 100644 --- a/components/producers/aggregator/Makefile +++ b/components/producers/aggregator/Makefile @@ -3,15 +3,17 @@ CONTAINER_REPO= DRACON_VERSION= SOURCE_CODE_REPO= +BUILD_ARCHITECTURE= DOCKER=docker container: - $(DOCKER) --debug build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ - --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/aggregator-parser \ + $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) \ + --build-arg EXECUTABLE_SRC_PATH=components/producers/aggregator/$(BUILD_ARCHITECTURE)/aggregator-parser \ --build-arg EXECUTABLE_TARGET_PATH=components/producers/aggregator/tagger \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --file ${BASE_IMAGE_PATH} ../../../bin 1>&2 + --platform "$(BUILD_ARCHITECTURE)" \ + --file "${BASE_IMAGE_PATH}" ../../../bin 1>&2 publish: $(DOCKER) push $(CONTAINER_REPO)/components/producers/tagger:$(DRACON_VERSION) 1>&2 diff --git a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile index 35e358a2b..a5847bfd9 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Dockerfile +++ b/components/producers/typescript-eslint/eslint-wrapper/Dockerfile @@ -1,10 +1,11 @@ ARG ESLINT_WRAPPER_BASE_IMAGE FROM ${ESLINT_WRAPPER_BASE_IMAGE:-node:lts} +ARG BUILD_ARCHITECTURE WORKDIR /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/eslintrc.js /home/node/workspace COPY components/producers/typescript-eslint/eslint-wrapper/package.json /home/node/workspace -COPY bin/components/producers/typescript-eslint/eslint-wrapper/eslint-wrapper-parser /home/node/workspace/ +COPY bin/components/producers/typescript-eslint/eslint-wrapper/${BUILD_ARCHITECTURE}/eslint-wrapper-parser /home/node/workspace/ RUN npm uninstall --save bcrypt &&\ npm install --save-dev \ @@ -14,4 +15,4 @@ RUN npm uninstall --save bcrypt &&\ eslint-plugin-no-unsanitized \ eslint-plugin-security-node -ENTRYPOINT [ "/home/node/workspace/eslint-wrapper-parser"] +ENTRYPOINT ["/home/node/workspace/eslint-wrapper-parser"] diff --git a/components/producers/typescript-eslint/eslint-wrapper/Makefile b/components/producers/typescript-eslint/eslint-wrapper/Makefile index a4e3c8ce0..845557f1b 100644 --- a/components/producers/typescript-eslint/eslint-wrapper/Makefile +++ b/components/producers/typescript-eslint/eslint-wrapper/Makefile @@ -2,6 +2,7 @@ CONTAINER_REPO= DRACON_VERSION= +BUILD_ARCHITECTURE= DOCKER=docker @@ -9,4 +10,6 @@ container: $(DOCKER) build --tag $(CONTAINER_REPO)/components/producers/typescript-eslint/eslint-wrapper:$(DRACON_VERSION) \ --file Dockerfile \ $$([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) ../../../.. 1>&2 + --build-arg ESLINT_WRAPPER_BASE_IMAGE=$(ESLINT_WRAPPER_BASE_IMAGE) \ + --build-arg BUILD_ARCHITECTURE=$(BUILD_ARCHITECTURE) \ + ../../../.. 1>&2 diff --git a/containers/Dockerfile.draconctl b/containers/Dockerfile.draconctl index 9d550ea61..05ef77304 100644 --- a/containers/Dockerfile.draconctl +++ b/containers/Dockerfile.draconctl @@ -1,9 +1,13 @@ ARG BASE_MIGRATION_IMAGE + FROM ${BASE_MIGRATION_IMAGE:-scratch} +ARG GOOS=linux +ARG GOARCH=amd64 + LABEL org.opencontainers.image.description="Draconctl is a command line tool for interacting with Dracon, you can find documentation for it at github.com/ocurity/dracon" -COPY ./bin/cmd/draconctl /bin/draconctl +COPY ./bin/cmd/${GOOS}/${GOARCH}/draconctl /bin/draconctl COPY ./pkg/enrichment/migrations /etc/dracon/migrations/enrichment ENTRYPOINT ["/bin/draconctl"] diff --git a/scripts/build_component_binary.sh b/scripts/build_component_binary.sh index 64d97e651..1dc3380ac 100755 --- a/scripts/build_component_binary.sh +++ b/scripts/build_component_binary.sh @@ -19,6 +19,11 @@ echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" | executable_src_path=$(dirname "${1}") executable_path=$(dirname $(dirname "${1}"))/"${executable}" -echo "building bin/${executable_path}/${executable}" > /dev/stderr +# Customised binary per OS/ARCH. +GOOS=${GOOS:-$(go env GOOS)} +GOARCH=${GOARCH:-$(go env GOARCH)} +out_bin_path="bin/${executable_src_path}/${GOOS}/${GOARCH}/${executable}" -go build -o "bin/${executable_src_path}/${executable}" "./${executable_src_path}/main.go" +echo "building $out_bin_path" > /dev/stderr + +CGO_ENABLED=0 go build -o $out_bin_path "./${executable_src_path}/main.go" diff --git a/scripts/build_component_container.sh b/scripts/build_component_container.sh index 67b057445..860ff4485 100755 --- a/scripts/build_component_container.sh +++ b/scripts/build_component_container.sh @@ -4,36 +4,45 @@ set -e; source ./scripts/util.sh +# Sanity check for not arguments being passed. if [ "$#" -eq 0 ] then - util::error "No directory provided to build" + util::error "No directory argument provided to build." exit 1 fi -executable=$(basename $(dirname "${1}")) +dir_name="$1" +GOOS="${GOOS}" +GOARCH="${GOARCH}" +build_architecture="${GOOS}/${GOARCH}" -echo "${1}" | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true -echo "${1}" | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true -echo "${1}" | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true +executable=$(basename $(dirname ${dir_name})) -executable_src_path=$(dirname "${1}") -executable_path=$(dirname "${1}")/"${executable}" +echo ${dir_name} | grep -Eq ^components/producers/.*$ && executable="${executable}-parser" || true +echo ${dir_name} | grep -Eq ^components/enrichers/.*$ && executable="${executable}" || true +echo ${dir_name} | grep -Eq ^components/consumers/.*$ && executable="${executable}" || true + +EXECUTABLE_SRC_PATH="$(dirname ${dir_name})/${build_architecture}/${executable}" +COMPONENT_PATH="$(dirname ${dir_name})" +EXECUTABLE_TARGET_PATH="${COMPONENT_PATH}/${executable}" BASE_IMAGE_PATH=$(realpath ${BASE_IMAGE_PATH:-./containers/Dockerfile.base}) -if make -C "${executable_src_path}" --no-print-directory --dry-run container >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run container >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet container BASE_IMAGE_PATH="${BASE_IMAGE_PATH}" CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" BUILD_ARCHITECTURE="${build_architecture}" else docker build \ - --build-arg EXECUTABLE_SRC_PATH=${executable_path} \ - --build-arg EXECUTABLE_TARGET_PATH=${executable_path} \ - -t "${CONTAINER_REPO}/${executable_src_path}:${DRACON_VERSION}" \ + --build-arg EXECUTABLE_SRC_PATH="${EXECUTABLE_SRC_PATH}" \ + --build-arg EXECUTABLE_TARGET_PATH="${EXECUTABLE_TARGET_PATH}" \ + --tag "${CONTAINER_REPO}/${COMPONENT_PATH}:${DRACON_VERSION}" \ $([ "${SOURCE_CODE_REPO}" != "" ] && echo "--label=org.opencontainers.image.source=${SOURCE_CODE_REPO}" ) \ - -f "${BASE_IMAGE_PATH}" ./bin + --file "${BASE_IMAGE_PATH}" \ + --platform "${build_architecture}" \ + ./bin fi -if make -C "${executable_src_path}" --no-print-directory --dry-run extras >/dev/null 2>&1 +if make -C "${COMPONENT_PATH}" --no-print-directory --dry-run extras >/dev/null 2>&1 then - make -C "${executable_src_path}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" + make -C "${COMPONENT_PATH}" --no-print-directory --quiet extras CONTAINER_REPO="${CONTAINER_REPO}" DRACON_VERSION="${DRACON_VERSION}" fi