Return image id after pulling helper images

Also refactor the pull helper image functions to support a single repository with multiple variant tags. Change-type: minor Signed-off-by: Kyle Harding <[email protected]>
balena-os · Feb 2, 2024 · 12ebad2 · 12ebad2
1 parent 8a15692
commit 12ebad2
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 73 deletions.
diff --git a/automation/include/balena-deploy.inc b/automation/include/balena-deploy.inc
@@ -146,13 +146,13 @@ balena_deploy_artifacts () {
 # $1: Device type name
 # $2: ESR flag
 # $3: S3 environmennt to deploy to (production | staging ) (defaults to production)
-# $4: S3 account namespace to use (defaults to resin)
+# $4: Helper image repository
 #
 balena_deploy_to_s3() {
 	local _device_type="${1}"
 	local _esr=${2}
 	local _deploy_to="${3:-"production"}"
-	local _namespace="${4}"
+	local _image_repo="${4}"
 	local _slug
 	local _artifact
 	local _s3_bucket
@@ -167,7 +167,7 @@ balena_deploy_to_s3() {
 	[ -z "${_device_type}" ] && echo "[balena_deploy_to_s3] Device type is required" && return 1
 	[ -z "${_s3_version_hostos}" ] && echo "[balena_deploy_to_s3] No hostOS version found" && return 1
 
-	[ -z "${_namespace}" ] && _namespace=${NAMESPACE:-"balena"}
+	[ -z "${_image_repo}" ] && _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 	[ -z "${_esr}" ] && _esr="${_esr:-"${ESR:-false}"}"
 	local _s3_deploy_dir="${device_dir}/deploy-s3"
 	local _s3_deploy_images_dir="$_s3_deploy_dir/$_slug/$_s3_version_hostos"
@@ -210,7 +210,9 @@ balena_deploy_to_s3() {
 
 	local _s3_cmd="s4cmd --access-key=${_s3_access_key} --secret-key=${_s3_secret_key}"
 	local _s3_sync_opts="--recursive --API-ACL=${_s3_policy}"
-	docker pull "${_namespace}/balena-img:6.20.26"
+	if ! balena_lib_docker_pull_helper_image "docker.io/balena/balena-img" "6.20.26" "" helper_image_id; then
+		return 1
+	fi
 	docker run --rm -t \
 		-e BASE_DIR=/host/images \
 		-e S3_CMD="$_s3_cmd" \
@@ -222,7 +224,7 @@ balena_deploy_to_s3() {
 		-e VERBOSE="${VERBOSE}" \
 		-e DEPLOYER_UID="$(id -u)" \
 		-e DEPLOYER_GID="$(id -g)" \
-		-v "$_s3_deploy_dir":/host/images "${_namespace}"/balena-img:6.20.26 /bin/sh -e -c ' \
+		-v "$_s3_deploy_dir":/host/images "${helper_image_id}" /bin/sh -e -c ' \
 			VERBOSE=${VERBOSE:-0}
 			[ "${VERBOSE}" = "verbose" ] && set -x
 			apt-get -y update
@@ -300,8 +302,8 @@ balena_deploy_block() {
 		fi
 	fi
 
-	NAMESPACE=${NAMESPACE:-balena}
-	if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-push-env" balena_yocto_scripts_revision; then
+	HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
+	if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "balena-push-env" helper_image_id; then
 		return 1
 	fi
 	docker run --rm -t \
@@ -325,9 +327,9 @@ balena_deploy_block() {
 		-v "${device_dir}":/work \
 		-v "${_work_dir}":/deploy \
 		--privileged \
-		"${NAMESPACE}"/balena-push-env:"${balena_yocto_scripts_revision}" /balena-deploy-block.sh
+		"${helper_image_id}" /balena-deploy-block.sh
 
-	balena_lib_docker_remove_helper_images "balena-push-env"
+	balena_lib_docker_remove_helper_images
 }
 
 # Initialize a compose file in the specified path

diff --git a/automation/include/balena-lib.inc b/automation/include/balena-lib.inc
@@ -6,9 +6,9 @@ VERBOSE=${VERBOSE:-0}
 
 include_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 device_dir=$( if cat /proc/self/cgroup | grep -q docker && [ -d "/work" ]; then cd "/work" && pwd; else cd "${include_dir}/../../.." && pwd; fi )
-BALENA_YOCTO_SCRIPTS_REVISION=$(cd "${include_dir}" && git rev-parse --short=7 HEAD || true)
 
-NAMESPACE="${NAMESPACE:-"balena"}"
+BALENA_YOCTO_SCRIPTS_VERSION=$(cd "${include_dir}" && head -n1 ../../VERSION)
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 # Return the latest tag of the device repository
 balena_lib_get_os_version() {
@@ -48,41 +48,35 @@ balena_lib_get_meta_balena_base_version() {
 
 # Pull a helper image building a matching version if required
 # Inputs:
-# $1: Dockerfile name
+# $1: Helper image repository
+# $2: Helper image version (defaults to balena-yocto-scripts version)
+# $3: Helper image variant (e.g. yocto-build-env)
+# $4: Return value variable name
 balena_lib_docker_pull_helper_image() {
-	local _dockerfile_name="$1"
-	local _image_name=""
-	local _image_prefix=""
-	local _retvalue="$2"
-	_image_name="${_dockerfile_name%".template"}"
-	_image_name="${_image_name#"Dockerfile_"}"
-	case ${_dockerfile_name} in
-		*template)
-			_image_prefix="${MACHINE}-"
-			DEVICE_ARCH=$(jq --raw-output '.arch' "$WORKSPACE/$MACHINE.json")
-			export DEVICE_ARCH
-			DEVICE_TYPE=${MACHINE}
-			export DEVICE_TYPE
-			;;
-	esac
+	local _image_repo="${1:-"${HELPER_IMAGE_REPO}"}"
+	local _image_version="${2:-"${BALENA_YOCTO_SCRIPTS_VERSION}"}"
+	local _image_variant="${3:-}"
+	local _retvalue="${4:-helper_image_id}"
+
+	local _image_tag="${_image_repo}":"${_image_version}"
+	local _image_id
 
-	if ! docker pull "${NAMESPACE}"/"${_image_prefix}""${_image_name}":"${BALENA_YOCTO_SCRIPTS_REVISION}"; then
-		JOB_NAME="${JOB_NAME}" DOCKERFILES="${_dockerfile_name}" "${include_dir}/../jenkins_build-containers.sh"
+	if [ -n "${_image_variant}" ]; then
+		_image_tag="${_image_tag}-${_image_variant}"
 	fi
-	eval "$_retvalue"='${BALENA_YOCTO_SCRIPTS_REVISION}'
+
+	docker pull "${_image_tag}"
+	_image_id="$(docker images --format "{{.ID}}" "${_image_tag}")"
+	eval "$_retvalue"='${_image_id}'
 }
 
-# Remove all versions of a  helper image
-# Inputs:
-# $1: Image name
-#
+# Remove all versions of the helper images
 balena_lib_docker_remove_helper_images() {
-	local _image_name="${1}"
 	local _image_ids
 	local _id
-	_image_ids=$(docker images "${NAMESPACE}/${_image_name}" --format "{{.ID}}")
+	_image_ids=$(docker images "${HELPER_IMAGE_REPO}" --format "{{.ID}}")
 	for _id in ${_image_ids}; do
-		docker rmi -f "${_id}" || true
+		docker rmi -f "${_id}" 2>/dev/null || true
 	done
 }
 

diff --git a/automation/jenkins_build-containers.sh b/automation/jenkins_build-containers.sh
diff --git a/automation/jenkins_build.sh b/automation/jenkins_build.sh
@@ -3,7 +3,7 @@
 [ "${VERBOSE}" = "verbose" ] && set -x
 set -e
 
-NAMESPACE=${NAMESPACE:-balena}
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 print_help() {
 	echo -e "Script options:\n\

diff --git a/automation/jenkins_generate_ami.sh b/automation/jenkins_generate_ami.sh
@@ -25,12 +25,12 @@ if [ "${deployTo}" = 'production' ]; then
     AWS_SECURITY_GROUP_ID="sg-057937f4d89d9d51c"
 fi
 
-NAMESPACE=${NAMESPACE:-balena}
+HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 # shellcheck disable=SC1091,SC2154
 source "${automation_dir}/include/balena-lib.inc"
 
-if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-generate-ami-env" balena_yocto_scripts_revision; then
+if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "yocto-generate-ami-env" helper_image_id; then
     exit 1
 fi
 
@@ -113,6 +113,6 @@ docker run --rm -t \
     -e AWS_SUBNET_ID="${AWS_SUBNET_ID}" \
     -e AWS_SECURITY_GROUP_ID="${AWS_SECURITY_GROUP_ID}" \
     -w "${WORKSPACE}" \
-    "${NAMESPACE}/balena-generate-ami-env:${balena_yocto_scripts_revision}" /balena-generate-ami.sh
+    "${helper_image_id}" /balena-generate-ami.sh
 
 rm -f "${PRELOADED_IMAGE}"
diff --git a/build/balena-build.sh b/build/balena-build.sh
@@ -59,7 +59,7 @@ balena_build_run_barys() {
 	local _docker_run_args="${9:-"--rm"}"
 	local _dl_dir
 	local _sstate_dir
-	local _namespace="${NAMESPACE:-"balena"}"
+	local _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}"
 
 	[ -z "${_device_type}" ] && echo "Device type is required"  && exit 1
 	[ -z "${_shared_dir}" ] && echo "Shared directory path is required"  && exit 1
@@ -86,11 +86,11 @@ balena_build_run_barys() {
 
 	"${DOCKER}" stop $BUILD_CONTAINER_NAME 2> /dev/null || true
 	"${DOCKER}" rm --volumes $BUILD_CONTAINER_NAME 2> /dev/null || true
-	if ! balena_lib_docker_pull_helper_image "Dockerfile_yocto-build-env" balena_yocto_scripts_revision; then
+	if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "yocto-build-env" helper_image_id; then
 		exit 1
 	fi
 	[ -z "${SSH_AUTH_SOCK}" ] && echo "No SSH_AUTH_SOCK in environment - private repositories won't be accessible to the builder" && SSH_AUTH_SOCK="/dev/null"
-	${DOCKER} run --rm ${__docker_run_args} \
+	${DOCKER} run --rm ${_docker_run_args} \
 		-v "${work_dir}":/work \
 		-v "${_dl_dir}":/yocto/shared-downloads \
 		-v "${_sstate_dir}":/yocto/shared-sstate \
@@ -103,7 +103,7 @@ balena_build_run_barys() {
 		-e API_ENV="${_api_env}" \
 		--name $BUILD_CONTAINER_NAME \
 		--privileged \
-		"${_namespace}"/yocto-build-env:"${balena_yocto_scripts_revision}" \
+		"${helper_image_id}" \
 		/prepare-and-start.sh \
 		--log \
 		--machine "${_device_type}" \
@@ -117,7 +117,7 @@ balena_build_run_barys() {
 		--rm-work
 
 	if [ "${_keep_helpers}" = "0" ]; then
-		balena_lib_docker_remove_helper_images "yocto-build-env"
+		balena_lib_docker_remove_helper_images
 	fi
 }