diff --git a/.github/workflows/yocto-build-deploy.yml b/.github/workflows/yocto-build-deploy.yml index ba4e28c0f..de5572543 100644 --- a/.github/workflows/yocto-build-deploy.yml +++ b/.github/workflows/yocto-build-deploy.yml @@ -200,6 +200,9 @@ jobs: # https://docs.yoctoproject.org/3.1.21/overview-manual/overview-manual-concepts.html#user-configuration # Create an autobuilder configuration file that is loaded before local.conf AUTO_CONF_FILE: "${{ github.workspace }}/build/conf/auto.conf" + SOURCE_MIRROR_REGION: ${{ vars.SOURCE_MIRROR_REGION || vars.AWS_REGION || 'us-east-1' }} + SOURCE_MIRROR_S3_URL: ${{ vars.SOURCE_MIRROR_S3_URL || 's3://yocto-72c1c258-81bb-11ef-b722-0efcede062c9/shared-downloads' }} + SOURCE_MIRROR_URL: ${{ vars.SOURCE_MIRROR_URL || 'https://yocto-72c1c258-81bb-11ef-b722-0efcede062c9.s3.us-east-1.amazonaws.com/shared-downloads/' }} outputs: os_version: ${{ steps.balena-lib.outputs.os_version }} @@ -462,6 +465,7 @@ jobs: BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} --bitbake-args --no-setscene" echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >>"${GITHUB_ENV}" + # FIXME: Remove this when we shut down the NFS server in early 2025 - name: Mount shared NFS cache if: vars.YOCTO_CACHE_HOST && contains(fromJSON(inputs.build-runs-on), 'self-hosted') continue-on-error: true @@ -474,6 +478,7 @@ jobs: sudo chown -R "$(id -u):$(id -g)" "${MOUNTPOINT}" sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${MOUNTPOINT}" -o fsc,nolock + # FIXME: Remove this when we shut down the NFS server in early 2025 # https://wiki.yoctoproject.org/wiki/Enable_sstate_cache # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-MIRRORS # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-PREMIRRORS @@ -505,24 +510,30 @@ jobs: EOF cat "${AUTO_CONF_FILE}" - # # FIXME: We should probably change this to MIRRORS:append instead of PREMIRRORS:prepend - # # to avoid using our S3 egress as much as possible? - # # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass - # # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass - # # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before - # # attempting to fetch it from the upstream specified in SRC_URI within each recipe. - # - name: Add S3 shared-downloads to PREMIRRORS - # env: - # SOURCE_MIRROR_URL: https://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}.s3.${{ vars.AWS_REGION || 'us-east-1' }}.amazonaws.com/shared-downloads/ - # run: | - # mkdir -p "$(dirname "${AUTO_CONF_FILE}")" - # cat <> "${AUTO_CONF_FILE}" + # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass + # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass + # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before + # attempting to fetch it from the upstream specified in SRC_URI within each recipe. + - name: Add S3 shared-downloads to MIRRORS + if: env.SOURCE_MIRROR_URL + env: + SOURCE_MIRROR_URL: ${{ env.SOURCE_MIRROR_URL }} + run: | + mkdir -p "$(dirname "${AUTO_CONF_FILE}")" + cat <> "${AUTO_CONF_FILE}" - # INHERIT += "own-mirrors" - # SOURCE_MIRROR_URL = "${SOURCE_MIRROR_URL}" + MIRRORS:append = "\\ + cvs://.*/.* ${SOURCE_MIRROR_URL} \\ + svn://.*/.* ${SOURCE_MIRROR_URL} \\ + git://.*/.* ${SOURCE_MIRROR_URL} \\ + hg://.*/.* ${SOURCE_MIRROR_URL} \\ + bzr://.*/.* ${SOURCE_MIRROR_URL} \\ + https?$://.*/.* ${SOURCE_MIRROR_URL} \\ + ftp://.*/.* ${SOURCE_MIRROR_URL} \\ + " - # EOF - # cat "${AUTO_CONF_FILE}" + EOF + cat "${AUTO_CONF_FILE}" # Use local S3 cache on self-hosted runners # https://github.com/tespkg/actions-cache @@ -549,11 +560,8 @@ jobs: key: ${{ inputs.machine }}-sstate-${{ github.sha }} restore-keys: | ${{ inputs.machine }}-sstate- - # FIXME: Include the shared-downloads directory with the sstate cache for now until - # we fully switch to AWS S3 for shared-downloads path: | ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate - ${{ github.workspace }}/shared/shared-downloads # All preperation complete before this step # Start building balenaOS @@ -619,11 +627,8 @@ jobs: region: local use-fallback: ${{ github.event.repository.private != true }} key: ${{ inputs.machine }}-sstate-${{ github.sha }} - # FIXME: Include the shared-downloads directory with the sstate cache for now until - # we fully switch to AWS S3 for shared-downloads path: | ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate - ${{ github.workspace }}/shared/shared-downloads # https://github.com/unfor19/install-aws-cli-action - name: Setup awscli @@ -639,29 +644,27 @@ jobs: # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664 mask-aws-account-id: false - # # Sync shared downloads to S3 to use as a sources mirror in case original sources are not available. - # # Exlude all directories and temp files as we only want the content and the .done files. - # # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html - # - name: Sync shared downloads to S3 - # # Do not publish shared downloads for pull_request_target events to prevent cache poisoning - # # Do not publish shared downloads for private device-types as the mirror is public-read - # if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' - # # Ignore errors for now, as we may have upload conflicts with other jobs - # continue-on-error: true - # env: - # SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads - # S3_ACL: public-read - # S3_SSE: AES256 - # # FIXME: This should be a a new bucket used only for shared-downloads (one for staging, one for production) - # S3_URL: "s3://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}/shared-downloads" - # S3_REGION: ${{ vars.AWS_REGION || 'us-east-1' }} - # # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks - # # created in the build container runtime. - # run: | - # sudo ln -sf "${{ github.workspace }}" /work - # ls -al "${SHARED_DOWNLOADS_DIR}/" - # aws s3 sync --sse="${S3_SSE}" --acl="${S3_ACL}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" \ - # --exclude "*/*" --exclude "*.tmp" --size-only --follow-symlinks --no-progress + # Sync shared downloads to S3 to use as a sources mirror in case original sources are not available. + # Exlude all directories and temp files as we only want the content and the .done files. + # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html + - name: Sync shared downloads to S3 + # Do not publish shared downloads for pull_request_target events to prevent cache poisoning + # Do not publish shared downloads for private device-types as the mirror is public-read + if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' && env.SOURCE_MIRROR_S3_URL + # Ignore errors for now, as we may have upload conflicts with other jobs + continue-on-error: true + env: + SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads + S3_SSE: AES256 + S3_URL: ${{ env.SOURCE_MIRROR_S3_URL }} + S3_REGION: ${{ env.SOURCE_MIRROR_REGION }} + # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks + # created in the build container runtime. + run: | + sudo ln -sf "${{ github.workspace }}" /work + du -cksh "${SHARED_DOWNLOADS_DIR}/*" + aws s3 sync --sse="${S3_SSE}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" \ + --exclude "*/*" --exclude "*.tmp" --size-only --follow-symlinks --no-progress # TODO: pre-install on self-hosted-runners # Needed by the yocto job to zip artifacts - Don't remove