diff --git a/automation/include/balena-deploy.inc b/automation/include/balena-deploy.inc index 850205b55..9057cc405 100644 --- a/automation/include/balena-deploy.inc +++ b/automation/include/balena-deploy.inc @@ -146,13 +146,13 @@ balena_deploy_artifacts () { # $1: Device type name # $2: ESR flag # $3: S3 environmennt to deploy to (production | staging ) (defaults to production) -# $4: S3 account namespace to use (defaults to resin) +# $4: Helper image repository # balena_deploy_to_s3() { local _device_type="${1}" local _esr=${2} local _deploy_to="${3:-"production"}" - local _namespace="${4}" + local _image_repo="${4}" local _slug local _artifact local _s3_bucket @@ -167,7 +167,7 @@ balena_deploy_to_s3() { [ -z "${_device_type}" ] && echo "[balena_deploy_to_s3] Device type is required" && return 1 [ -z "${_s3_version_hostos}" ] && echo "[balena_deploy_to_s3] No hostOS version found" && return 1 - [ -z "${_namespace}" ] && _namespace=${NAMESPACE:-"balena"} + [ -z "${_image_repo}" ] && _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" [ -z "${_esr}" ] && _esr="${_esr:-"${ESR:-false}"}" local _s3_deploy_dir="${device_dir}/deploy-s3" local _s3_deploy_images_dir="$_s3_deploy_dir/$_slug/$_s3_version_hostos" @@ -210,7 +210,9 @@ balena_deploy_to_s3() { local _s3_cmd="s4cmd --access-key=${_s3_access_key} --secret-key=${_s3_secret_key}" local _s3_sync_opts="--recursive --API-ACL=${_s3_policy}" - docker pull "${_namespace}/balena-img:6.20.26" + if ! balena_lib_docker_pull_helper_image "docker.io/balena/balena-img" "6.20.26" "" helper_image_id; then + return 1 + fi docker run --rm -t \ -e BASE_DIR=/host/images \ -e S3_CMD="$_s3_cmd" \ @@ -222,7 +224,7 @@ balena_deploy_to_s3() { -e VERBOSE="${VERBOSE}" \ -e DEPLOYER_UID="$(id -u)" \ -e DEPLOYER_GID="$(id -g)" \ - -v "$_s3_deploy_dir":/host/images "${_namespace}"/balena-img:6.20.26 /bin/sh -e -c ' \ + -v "$_s3_deploy_dir":/host/images "${helper_image_id}" /bin/sh -e -c ' \ VERBOSE=${VERBOSE:-0} [ "${VERBOSE}" = "verbose" ] && set -x apt-get -y update @@ -300,8 +302,8 @@ balena_deploy_block() { fi fi - NAMESPACE=${NAMESPACE:-balena} - if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-push-env" balena_yocto_scripts_revision; then + HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" + if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "balena-push-env" helper_image_id; then return 1 fi docker run --rm -t \ @@ -325,9 +327,9 @@ balena_deploy_block() { -v "${device_dir}":/work \ -v "${_work_dir}":/deploy \ --privileged \ - "${NAMESPACE}"/balena-push-env:"${balena_yocto_scripts_revision}" /balena-deploy-block.sh + "${helper_image_id}" /balena-deploy-block.sh - balena_lib_docker_remove_helper_images "balena-push-env" + balena_lib_docker_remove_helper_images } # Initialize a compose file in the specified path diff --git a/automation/include/balena-lib.inc b/automation/include/balena-lib.inc index 5bed29db9..65cbdd34a 100644 --- a/automation/include/balena-lib.inc +++ b/automation/include/balena-lib.inc @@ -6,9 +6,9 @@ VERBOSE=${VERBOSE:-0} include_dir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) device_dir=$( if cat /proc/self/cgroup | grep -q docker && [ -d "/work" ]; then cd "/work" && pwd; else cd "${include_dir}/../../.." && pwd; fi ) -BALENA_YOCTO_SCRIPTS_REVISION=$(cd "${include_dir}" && git rev-parse --short=7 HEAD || true) -NAMESPACE="${NAMESPACE:-"balena"}" +BALENA_YOCTO_SCRIPTS_VERSION=$(cd "${include_dir}" && head -n1 ../../VERSION) +HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" # Return the latest tag of the device repository balena_lib_get_os_version() { @@ -48,41 +48,35 @@ balena_lib_get_meta_balena_base_version() { # Pull a helper image building a matching version if required # Inputs: -# $1: Dockerfile name +# $1: Helper image repository +# $2: Helper image version (defaults to balena-yocto-scripts version) +# $3: Helper image variant (e.g. yocto-build-env) +# $4: Return value variable name balena_lib_docker_pull_helper_image() { - local _dockerfile_name="$1" - local _image_name="" - local _image_prefix="" - local _retvalue="$2" - _image_name="${_dockerfile_name%".template"}" - _image_name="${_image_name#"Dockerfile_"}" - case ${_dockerfile_name} in - *template) - _image_prefix="${MACHINE}-" - DEVICE_ARCH=$(jq --raw-output '.arch' "$WORKSPACE/$MACHINE.json") - export DEVICE_ARCH - DEVICE_TYPE=${MACHINE} - export DEVICE_TYPE - ;; - esac + local _image_repo="${1:-"${HELPER_IMAGE_REPO}"}" + local _image_version="${2:-"${BALENA_YOCTO_SCRIPTS_VERSION}"}" + local _image_variant="${3:-}" + local _retvalue="${4:-helper_image_id}" + + local _image_tag="${_image_repo}":"${_image_version}" + local _image_id - if ! docker pull "${NAMESPACE}"/"${_image_prefix}""${_image_name}":"${BALENA_YOCTO_SCRIPTS_REVISION}"; then - JOB_NAME="${JOB_NAME}" DOCKERFILES="${_dockerfile_name}" "${include_dir}/../jenkins_build-containers.sh" + if [ -n "${_image_variant}" ]; then + _image_tag="${_image_tag}-${_image_variant}" fi - eval "$_retvalue"='${BALENA_YOCTO_SCRIPTS_REVISION}' + + docker pull "${_image_tag}" + _image_id="$(docker images --format "{{.ID}}" "${_image_tag}")" + eval "$_retvalue"='${_image_id}' } -# Remove all versions of a helper image -# Inputs: -# $1: Image name -# +# Remove all versions of the helper images balena_lib_docker_remove_helper_images() { - local _image_name="${1}" local _image_ids local _id - _image_ids=$(docker images "${NAMESPACE}/${_image_name}" --format "{{.ID}}") + _image_ids=$(docker images "${HELPER_IMAGE_REPO}" --format "{{.ID}}") for _id in ${_image_ids}; do - docker rmi -f "${_id}" || true + docker rmi -f "${_id}" 2>/dev/null || true done } diff --git a/automation/jenkins_build-containers.sh b/automation/jenkins_build-containers.sh deleted file mode 100755 index 4b0fc10f9..000000000 --- a/automation/jenkins_build-containers.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash - -set -ev - -[ -z "${DOCKERFILES}" ] && DOCKERFILES=( Dockerfile_yocto-block-build-env Dockerfile_yocto-build-env Dockerfile_balena-push-env Dockerfile_balena-generate-ami-env ) - -SCRIPTPATH=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -REVISION=$(cd "${SCRIPTPATH}" && git rev-parse --short=7 HEAD) -NAMESPACE=${NAMESPACE:-balena} - -source "${SCRIPTPATH}/include/balena-lib.inc" - -balena_lib_dockerhub_login - -for DOCKERFILE in "${DOCKERFILES[@]}" -do - REPO_NAME=${DOCKERFILE#"Dockerfile_"} - # Build - docker build --pull --no-cache --tag ${NAMESPACE}/${REPO_NAME}:${REVISION} -f ${SCRIPTPATH}/${DOCKERFILE} ${SCRIPTPATH} - - # Tag - docker tag ${NAMESPACE}/${REPO_NAME}:${REVISION} ${NAMESPACE}/${REPO_NAME}:latest - - # Push - docker push ${NAMESPACE}/${REPO_NAME}:${REVISION} - docker push ${NAMESPACE}/${REPO_NAME}:latest -done diff --git a/automation/jenkins_build.sh b/automation/jenkins_build.sh index 6bbe768e7..4a63d0f83 100755 --- a/automation/jenkins_build.sh +++ b/automation/jenkins_build.sh @@ -3,7 +3,7 @@ [ "${VERBOSE}" = "verbose" ] && set -x set -e -NAMESPACE=${NAMESPACE:-balena} +HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" print_help() { echo -e "Script options:\n\ diff --git a/automation/jenkins_generate_ami.sh b/automation/jenkins_generate_ami.sh index 088a1db36..27033b8bc 100755 --- a/automation/jenkins_generate_ami.sh +++ b/automation/jenkins_generate_ami.sh @@ -25,12 +25,12 @@ if [ "${deployTo}" = 'production' ]; then AWS_SECURITY_GROUP_ID="sg-057937f4d89d9d51c" fi -NAMESPACE=${NAMESPACE:-balena} +HELPER_IMAGE_REPO="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" # shellcheck disable=SC1091,SC2154 source "${automation_dir}/include/balena-lib.inc" -if ! balena_lib_docker_pull_helper_image "Dockerfile_balena-generate-ami-env" balena_yocto_scripts_revision; then +if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "yocto-generate-ami-env" helper_image_id; then exit 1 fi @@ -113,6 +113,6 @@ docker run --rm -t \ -e AWS_SUBNET_ID="${AWS_SUBNET_ID}" \ -e AWS_SECURITY_GROUP_ID="${AWS_SECURITY_GROUP_ID}" \ -w "${WORKSPACE}" \ - "${NAMESPACE}/balena-generate-ami-env:${balena_yocto_scripts_revision}" /balena-generate-ami.sh + "${helper_image_id}" /balena-generate-ami.sh rm -f "${PRELOADED_IMAGE}" diff --git a/build/balena-build.sh b/build/balena-build.sh index 9759fbeec..b2193e6ec 100755 --- a/build/balena-build.sh +++ b/build/balena-build.sh @@ -59,7 +59,7 @@ balena_build_run_barys() { local _docker_run_args="${9:-"--rm"}" local _dl_dir local _sstate_dir - local _namespace="${NAMESPACE:-"balena"}" + local _image_repo="${HELPER_IMAGE_REPO:-"ghcr.io/balena-os/balena-yocto-scripts"}" [ -z "${_device_type}" ] && echo "Device type is required" && exit 1 [ -z "${_shared_dir}" ] && echo "Shared directory path is required" && exit 1 @@ -86,11 +86,11 @@ balena_build_run_barys() { "${DOCKER}" stop $BUILD_CONTAINER_NAME 2> /dev/null || true "${DOCKER}" rm --volumes $BUILD_CONTAINER_NAME 2> /dev/null || true - if ! balena_lib_docker_pull_helper_image "Dockerfile_yocto-build-env" balena_yocto_scripts_revision; then + if ! balena_lib_docker_pull_helper_image "${HELPER_IMAGE_REPO}" "" "yocto-build-env" helper_image_id; then exit 1 fi [ -z "${SSH_AUTH_SOCK}" ] && echo "No SSH_AUTH_SOCK in environment - private repositories won't be accessible to the builder" && SSH_AUTH_SOCK="/dev/null" - ${DOCKER} run --rm ${__docker_run_args} \ + ${DOCKER} run --rm ${_docker_run_args} \ -v "${work_dir}":/work \ -v "${_dl_dir}":/yocto/shared-downloads \ -v "${_sstate_dir}":/yocto/shared-sstate \ @@ -103,7 +103,7 @@ balena_build_run_barys() { -e API_ENV="${_api_env}" \ --name $BUILD_CONTAINER_NAME \ --privileged \ - "${_namespace}"/yocto-build-env:"${balena_yocto_scripts_revision}" \ + "${helper_image_id}" \ /prepare-and-start.sh \ --log \ --machine "${_device_type}" \ @@ -117,7 +117,7 @@ balena_build_run_barys() { --rm-work if [ "${_keep_helpers}" = "0" ]; then - balena_lib_docker_remove_helper_images "yocto-build-env" + balena_lib_docker_remove_helper_images fi }