From 07e02aef5af44f7d96e4f491ba8331a42214f654 Mon Sep 17 00:00:00 2001 From: Kyle Harding Date: Tue, 17 Dec 2024 12:46:14 -0500 Subject: [PATCH] Use dedicated AWS S3 buckets for download mirrors Change-type: minor Signed-off-by: Kyle Harding --- .github/workflows/yocto-build-deploy.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index 75eb43ab7..42dd8cd16 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -200,6 +200,9 @@ jobs: # https://docs.yoctoproject.org/3.1.21/overview-manual/overview-manual-concepts.html#user-configuration # Create an autobuilder configuration file that is loaded before local.conf AUTO_CONF_FILE: "${{ github.workspace }}/build/conf/auto.conf" + SOURCE_MIRROR_REGION: ${{ vars.SOURCE_MIRROR_REGION || vars.AWS_REGION || 'us-east-1' }} + SOURCE_MIRROR_S3_URL: ${{ vars.SOURCE_MIRROR_S3_URL || 's3://yocto-72c1c258-81bb-11ef-b722-0efcede062c9/shared-downloads' }} + SOURCE_MIRROR_URL: ${{ vars.SOURCE_MIRROR_URL || 'https://yocto-72c1c258-81bb-11ef-b722-0efcede062c9.s3.us-east-1.amazonaws.com/shared-downloads/' }} outputs: os_version: ${{ steps.balena-lib.outputs.os_version }} @@ -462,6 +465,7 @@ jobs: BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} --bitbake-args --no-setscene" echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >>"${GITHUB_ENV}" + # FIXME: Remove this when we shut down the NFS server in early 2025 - name: Mount shared NFS cache if: vars.YOCTO_CACHE_HOST && contains(fromJSON(inputs.build-runs-on), 'self-hosted') continue-on-error: true @@ -474,6 +478,7 @@ jobs: sudo chown -R "$(id -u):$(id -g)" "${MOUNTPOINT}" sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${MOUNTPOINT}" -o fsc,nolock + # FIXME: Remove this when we shut down the NFS server in early 2025 # https://wiki.yoctoproject.org/wiki/Enable_sstate_cache # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-MIRRORS # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-PREMIRRORS @@ -512,8 +517,9 @@ jobs: # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before # attempting to fetch it from the upstream specified in SRC_URI within each recipe. - name: Add S3 shared-downloads to PREMIRRORS + if: env.SOURCE_MIRROR_URL env: - SOURCE_MIRROR_URL: https://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}.s3.${{ vars.AWS_REGION || 'us-east-1' }}.amazonaws.com/shared-downloads/ + SOURCE_MIRROR_URL: ${{ env.SOURCE_MIRROR_URL }} run: | mkdir -p "$(dirname "${AUTO_CONF_FILE}")" cat <> "${AUTO_CONF_FILE}" @@ -639,15 +645,14 @@ jobs: - name: Sync shared downloads to S3 # Do not publish shared downloads for pull_request_target events to prevent cache poisoning # Do not publish shared downloads for private device-types as the mirror is public-read - if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' + if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' && env.SOURCE_MIRROR_S3_URL # Ignore errors for now, as we may have upload conflicts with other jobs continue-on-error: true env: SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads S3_SSE: AES256 - # FIXME: This should be a a new bucket used only for shared-downloads (one for staging, one for production) - S3_URL: "s3://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}/shared-downloads" - S3_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} + S3_URL: ${{ env.SOURCE_MIRROR_S3_URL }} + S3_REGION: ${{ env.SOURCE_MIRROR_REGION }} # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks # created in the build container runtime. run: |