Return image id after pulling helper images

Also refactor the pull helper image functions to
support a single repository with multiple variant tags.

Change-type: minor
Signed-off-by: Kyle Harding <[email protected]>

automation/Dockerfile_balena-generate-ami-env

@@ -1,9 +1,10 @@
-ARG NAMESPACE="balena"
-ARG TAG="latest"
-FROM ${NAMESPACE}/balena-push-env:${TAG}
+ARG HELPER_IMAGE_REPO="ghcr.io/balena-os/balena-yocto-scripts"
+ARG TAG=master
+FROM ${HELPER_IMAGE_REPO}:${TAG}-balena-push-env
 
 RUN apt-get update && apt-get install -y python3-pip udev && rm -rf /var/lib/apt/lists/*
 RUN pip3 install awscli
 
 COPY include/balena-api.inc include/balena-lib.inc entry_scripts/balena-generate-ami.sh /
 WORKDIR /
+∂

automation/Dockerfile_yocto-block-build-env

@@ -1,6 +1,6 @@
-ARG NAMESPACE="balena"
-ARG TAG="latest"
-FROM ${NAMESPACE}/balena-push-env:${TAG}
+ARG HELPER_IMAGE_REPO="ghcr.io/balena-os/balena-yocto-scripts"
+ARG TAG=master
+FROM ${HELPER_IMAGE_REPO}:${TAG}-balena-push-env
 
 RUN apt-get update && apt-get install -y git make automake gcc libtool libtool-bin pkg-config libarchive-dev libcurl4-openssl-dev libssl-dev libgpgme11-dev && rm -rf /var/lib/apt/lists/*
 

automation/entry_scripts/balena-build-block.sh

@@ -42,9 +42,9 @@ balena_docker_wait
 
 # Only support overlay images for the time being. Labels to be parametrized from contract in future.
 cat << 'EOF' > ${TMPDIR}/Dockerfile
-ARG NAMESPACE=balena
-ARG TAG=latest
-FROM ${NAMESPACE}/yocto-block-build-env:${TAG} AS builder
+ARG HELPER_IMAGE_REPO="ghcr.io/balena-os/balena-yocto-scripts"
+ARG TAG=master
+FROM ${HELPER_IMAGE_REPO}:${TAG}-balena-push-env
 ARG PACKAGES
 ARG ARCH_LIST
 ARG FEED_URL="file:/ipk"
@@ -89,7 +89,7 @@ pushd "${TMPDIR}"
 # Clean local docker of labelled hostos images
 docker rmi -f $(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --format "{{.ID}}" | tr '\n' ' ') 2> /dev/null || true
 
-if balena build --logs --nocache --deviceType "${MACHINE}" --arch "${ARCH}" --buildArg PACKAGES="${PACKAGES}" --buildArg ARCH_LIST="${ARCH_LIST}" --buildArg NAMESPACE="${NAMESPACE:-balena}"; then
+if balena build --logs --nocache --deviceType "${MACHINE}" --arch "${ARCH}" --buildArg PACKAGES="${PACKAGES}" --buildArg ARCH_LIST="${ARCH_LIST}" --buildArg HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"; then
 	image_id=$(docker images --filter "label=${BALENA_HOSTOS_BLOCK_CLASS}" --format "{{.ID}}")
 	mkdir -p "${WORKSPACE}/deploy-jenkins"
 	docker save "${image_id}" > "${WORKSPACE}/deploy-jenkins/${APPNAME}-${RELEASE_VERSION}.docker"

automation/include/balena-deploy.inc

@@ -146,28 +146,27 @@ balena_deploy_artifacts () {
 # $1: Device type name
 # $2: ESR flag
 # $3: S3 environmennt to deploy to (production | staging ) (defaults to production)
-# $4: S3 account namespace to use (defaults to resin)
 #
 balena_deploy_to_s3() {
 	local _device_type="${1}"
 	local _esr=${2}
 	local _deploy_to="${3:-"production"}"
-	local _namespace="${4}"
 	local _slug
 	local _artifact
 	local _s3_bucket
 	local _s3_bucket_prefix
 	local _s3_bucket_suffix
 	local _is_private
 	local _s3_version_hostos="$(balena_lib_get_os_version)"
+	local _image_id
 
 	_slug=$(balena_lib_get_slug "${_device_type}")
 	_artifact=$(balena_lib_get_deploy_artifact "${_device_type}")
 
 	[ -z "${_device_type}" ] && echo "[balena_deploy_to_s3] Device type is required" && return 1
 	[ -z "${_s3_version_hostos}" ] && echo "[balena_deploy_to_s3] No hostOS version found" && return 1
 
-	[ -z "${_namespace}" ] && _namespace=${NAMESPACE:-"balena"}
+	[ -z "${_image_repo}" ] && _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 	[ -z "${_esr}" ] && _esr="${_esr:-"${ESR:-false}"}"
 	local _s3_deploy_dir="${device_dir}/deploy-s3"
 	local _s3_deploy_images_dir="$_s3_deploy_dir/$_slug/$_s3_version_hostos"
@@ -210,7 +209,10 @@ balena_deploy_to_s3() {
 
 	local _s3_cmd="s4cmd --access-key=${_s3_access_key} --secret-key=${_s3_secret_key}"
 	local _s3_sync_opts="--recursive --API-ACL=${_s3_policy}"
-	docker pull "${_namespace}/balena-img:6.20.26"
+	HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
+	if ! balena_lib_docker_pull_helper_image "balena-img-s3" _image_id; then
+		return 1
+	fi
 	docker run --rm -t \
 		-e BASE_DIR=/host/images \
 		-e S3_CMD="$_s3_cmd" \
@@ -222,7 +224,7 @@ balena_deploy_to_s3() {
 		-e VERBOSE="${VERBOSE}" \
 		-e DEPLOYER_UID="$(id -u)" \
 		-e DEPLOYER_GID="$(id -g)" \
-		-v "$_s3_deploy_dir":/host/images "${_namespace}"/balena-img:6.20.26 /bin/sh -e -c ' \
+		-v "$_s3_deploy_dir":/host/images "${_image_id}" /bin/sh -e -c ' \
 			VERBOSE=${VERBOSE:-0}
 			[ "${VERBOSE}" = "verbose" ] && set -x
 			apt-get -y update
@@ -282,6 +284,7 @@ balena_deploy_block() {
 	local _balenaCloudPassword="${11:-"${balenaCloudPassword}"}"
 	local _esr="${12}"
 	local _api_env=$(balena_lib_environment)
+	local _image_id
 
 	[ -z "${_device_type}" ] && echo "Device type is required" && return
 
@@ -300,8 +303,8 @@ balena_deploy_block() {
 		fi
 	fi
 
-	NAMESPACE=${NAMESPACE:-balena}
-	if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-push-env" balena_yocto_scripts_revision; then
+	HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
+	if ! balena_lib_docker_pull_helper_image "balena-push-env" _image_id; then
 		return 1
 	fi
 	docker run --rm -t \
@@ -325,9 +328,9 @@ balena_deploy_block() {
 		-v "${device_dir}":/work \
 		-v "${_work_dir}":/deploy \
 		--privileged \
-		"${NAMESPACE}"/balena-push-env:"${balena_yocto_scripts_revision}" /balena-deploy-block.sh
+		"${_image_id}" /balena-deploy-block.sh
 
-	balena_lib_docker_remove_helper_images "balena-push-env"
+	balena_lib_docker_remove_helper_images
 }
 
 # Builds and deploys the specified block to BalenaCloud
@@ -344,31 +347,31 @@ balena_build_block() {
 	local _packages="${3:-${PACKAGES}}"
 	local _balenaos_account="${4:-balena_os}"
 	local _api_env="${5:-$(balena_lib_environment)}"
+	local _image_id
 
 	[ -z "${_appName}" ] && echo "App name is required" && return
 	[ -z "${_device_type}" ] && echo "Device type is required" && return
 	[ -z "${_packages}" ] && echo "Package list is required" && return
 
-	if ! balena_lib_docker_pull_helper_image "Dockerfile_yocto-block-build-env" balena_yocto_scripts_revision; then
+	if ! balena_lib_docker_pull_helper_image "block-build-env" _image_id; then
 		return 1
 	fi
 	docker run --rm -t \
 		-e APPNAME="${_appName}" \
-		-e NAMESPACE="${NAMESPACE:-balena}" \
+		-e HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-ghcr.io/balena-os/balena-yocto-scripts}" \
 		-e RELEASE_VERSION="$(balena_lib_get_os_version)" \
 		-e PACKAGES="${_packages}" \
 		-e VERBOSE="${VERBOSE}" \
 		-e MACHINE="${_device_type}" \
-		-e TAG="${balena_yocto_scripts_revision}" \
 		-e WORKSPACE=/work \
 		-e balenaCloudEmail="${balenaCloudEmail}" \
 		-e balenaCloudPassword="${balenaCloudPassword}" \
 		-e ESR="${ESR}" \
 		-v "${WORKSPACE:-"${PWD}"}":/work \
 		--privileged \
-		"${NAMESPACE}"/yocto-block-build-env:"${balena_yocto_scripts_revision}" /balena-build-block.sh
+		"${_image_id}" /balena-build-block.sh
 
-	balena_lib_docker_remove_helper_images "yocto-block-build-env"
+	balena_lib_docker_remove_helper_images
 }
 
 # Initialize a compose file in the specified path

automation/include/balena-lib.inc

@@ -6,9 +6,9 @@ VERBOSE=${VERBOSE:-0}
 
 include_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 device_dir=$( if cat /proc/self/cgroup | grep -q docker && [ -d "/work" ]; then cd "/work" && pwd; else cd "${include_dir}/../../.." && pwd; fi )
-BALENA_YOCTO_SCRIPTS_REVISION=$(cd "${include_dir}" && git rev-parse --short=7 HEAD || true)
 
-NAMESPACE="${NAMESPACE:-"balena"}"
+BALENA_YOCTO_SCRIPTS_VERSION=$(cd "${include_dir}" && head -n1 ../../VERSION)
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 # Return the latest tag of the device repository
 balena_lib_get_os_version() {
@@ -48,41 +48,26 @@ balena_lib_get_meta_balena_base_version() {
 
 # Pull a helper image building a matching version if required
 # Inputs:
-# $1: Dockerfile name
+# $1: Image suffix (e.g. yocto-build-env)
+# $2: Return value variable name
 balena_lib_docker_pull_helper_image() {
-	local _dockerfile_name="$1"
-	local _image_name=""
-	local _image_prefix=""
+	local _image_suffix="$1"
 	local _retvalue="$2"
-	_image_name="${_dockerfile_name%".template"}"
-	_image_name="${_image_name#"Dockerfile_"}"
-	case ${_dockerfile_name} in
-		*template)
-			_image_prefix="${MACHINE}-"
-			DEVICE_ARCH=$(jq --raw-output '.arch' "$WORKSPACE/$MACHINE.json")
-			export DEVICE_ARCH
-			DEVICE_TYPE=${MACHINE}
-			export DEVICE_TYPE
-			;;
-	esac
+	local _image_tag="${HELPER_IMAGE_REPO}":"${BALENA_YOCTO_SCRIPTS_VERSION}"-"${_image_suffix}"
+	local _image_id
 
-	if ! docker pull "${NAMESPACE}"/"${_image_prefix}""${_image_name}":"${BALENA_YOCTO_SCRIPTS_REVISION}"; then
-		JOB_NAME="${JOB_NAME}" DOCKERFILES="${_dockerfile_name}" "${include_dir}/../jenkins_build-containers.sh"
-	fi
-	eval "$_retvalue"='${BALENA_YOCTO_SCRIPTS_REVISION}'
+	docker pull "${_image_tag}"
+	_image_id="$(docker images --format "{{.ID}}" "${_image_tag}")"
+	eval "$_retvalue"='${_image_id}'
 }
 
-# Remove all versions of a  helper image
-# Inputs:
-# $1: Image name
-#
+# Remove all versions of the helper images
 balena_lib_docker_remove_helper_images() {
-	local _image_name="${1}"
 	local _image_ids
 	local _id
-	_image_ids=$(docker images "${NAMESPACE}/${_image_name}" --format "{{.ID}}")
+	_image_ids=$(docker images "${HELPER_IMAGE_REPO}" --format "{{.ID}}")
 	for _id in ${_image_ids}; do
-		docker rmi -f "${_id}" || true
+		docker rmi -f "${_id}" 2>/dev/null || true
 	done
 }
 

automation/jenkins_build-blocks.sh

@@ -17,7 +17,7 @@ Usage: ${script_name} [OPTIONS]
     -b HostOS block names
     -p Deploy as final version
     -t Balena API token
-    -n Registry namespace
+    -n Helper image repository
     -s Shared build directory
     -c Balena account (defaults to balena_os)
     -h Display usage
@@ -98,7 +98,7 @@ main() {
 	local _device_type
 	local _api_env
 	local _token
-	local _namespace
+	local _image_repo
 	local _shared_dir
 	local _hostos_blocks
 	local _balenaos_account
@@ -115,7 +115,7 @@ main() {
 				a) _api_env="${OPTARG}";;
 				b) _blocks="${OPTARG}";;
 				t) _token="${OPTARG}";;
-				n) _namespace="${OPTARG}" ;;
+				n) _image_repo="${OPTARG}" ;;
 				s) _shared_dir="${OPTARG}" ;;
 				c) _balenaos_account="${OPTARG}" ;;
 				e) _esr=true ;;
@@ -134,7 +134,7 @@ main() {
 		[ -z "${_shared_dir}" ] && echo "Shared directory is required" && exit 1
 		_blocks="${_blocks:-"${hostOSBlocks}"}"
 		[ -z "${_blocks}" ] && echo "No block names provided - nothing to do" && exit 1
-		[ -n "${_namespace}" ] && echo "Setting registry account to ${_namespace}" && export NAMESPACE=${_namespace}
+		[ -n "${_image_repo}" ] && echo "Setting helper image repository to ${_image_repo}" && export HELPER_IMAGE_REPO=${_image_repo}
 		_balenaos_account=${_balenaos_account:-balena_os}
 
 		_hostos_blocks=$(__build_hostos_blocks "${_device_type}" "${_shared_dir}" "${_blocks}" "${_api_env}" "${_balenaos_account}" "${_final}")

automation/jenkins_build.sh

@@ -3,7 +3,7 @@
 [ "${VERBOSE}" = "verbose" ] && set -x
 set -e
 
-NAMESPACE=${NAMESPACE:-balena}
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 print_help() {
 	echo -e "Script options:\n\

automation/jenkins_generate_ami.sh

@@ -25,12 +25,12 @@ if [ "${deployTo}" = 'production' ]; then
     AWS_SECURITY_GROUP_ID="sg-057937f4d89d9d51c"
 fi
 
-NAMESPACE=${NAMESPACE:-balena}
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 # shellcheck disable=SC1091,SC2154
 source "${automation_dir}/include/balena-lib.inc"
 
-if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-generate-ami-env" balena_yocto_scripts_revision; then
+if ! balena_lib_docker_pull_helper_image "generate-ami-env" image_id; then
     exit 1
 fi
 
@@ -113,6 +113,6 @@ docker run --rm -t \
     -e AWS_SUBNET_ID="${AWS_SUBNET_ID}" \
     -e AWS_SECURITY_GROUP_ID="${AWS_SECURITY_GROUP_ID}" \
     -w "${WORKSPACE}" \
-    "${NAMESPACE}/balena-generate-ami-env:${balena_yocto_scripts_revision}" /balena-generate-ami.sh
+    "${image_id}" /balena-generate-ami.sh
 
 rm -f "${PRELOADED_IMAGE}"

build/balena-build.sh

@@ -59,7 +59,8 @@ balena_build_run_barys() {
 	local _docker_run_args="${9:-"--rm"}"
 	local _dl_dir
 	local _sstate_dir
-	local _namespace="${NAMESPACE:-"balena"}"
+	local _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
+	local _image_id
 
 	[ -z "${_device_type}" ] && echo "Device type is required"  && exit 1
 	[ -z "${_shared_dir}" ] && echo "Shared directory path is required"  && exit 1
@@ -86,11 +87,11 @@ balena_build_run_barys() {
 
 	"${DOCKER}" stop $BUILD_CONTAINER_NAME 2> /dev/null || true
 	"${DOCKER}" rm --volumes $BUILD_CONTAINER_NAME 2> /dev/null || true
-	if ! balena_lib_docker_pull_helper_image "Dockerfile_yocto-build-env" balena_yocto_scripts_revision; then
+	if ! balena_lib_docker_pull_helper_image "yocto-build-env" _image_id; then
 		exit 1
 	fi
 	[ -z "${SSH_AUTH_SOCK}" ] && echo "No SSH_AUTH_SOCK in environment - private repositories won't be accessible to the builder" && SSH_AUTH_SOCK="/dev/null"
-	${DOCKER} run --rm ${__docker_run_args} \
+	${DOCKER} run --rm ${_docker_run_args} \
 		-v "${work_dir}":/work \
 		-v "${_dl_dir}":/yocto/shared-downloads \
 		-v "${_sstate_dir}":/yocto/shared-sstate \
@@ -103,7 +104,7 @@ balena_build_run_barys() {
 		-e API_ENV="${_api_env}" \
 		--name $BUILD_CONTAINER_NAME \
 		--privileged \
-		"${_namespace}"/yocto-build-env:"${balena_yocto_scripts_revision}" \
+		"${_image_id}" \
 		/prepare-and-start.sh \
 		--log \
 		--machine "${_device_type}" \
@@ -117,7 +118,7 @@ balena_build_run_barys() {
 		--rm-work
 
 	if [ "${_keep_helpers}" = "0" ]; then
-		balena_lib_docker_remove_helper_images "yocto-build-env"
+		balena_lib_docker_remove_helper_images
 	fi
 }