Use dedicated AWS S3 buckets for download mirrors

.github/workflows/yocto-build-deploy.yml

@@ -200,6 +200,9 @@ jobs:
       # https://docs.yoctoproject.org/3.1.21/overview-manual/overview-manual-concepts.html#user-configuration
       # Create an autobuilder configuration file that is loaded before local.conf
       AUTO_CONF_FILE: "${{ github.workspace }}/build/conf/auto.conf"
+      SOURCE_MIRROR_REGION: ${{ vars.SOURCE_MIRROR_REGION || vars.AWS_REGION || 'us-east-1' }}
+      SOURCE_MIRROR_S3_URL: ${{ vars.SOURCE_MIRROR_S3_URL || 's3://yocto-72c1c258-81bb-11ef-b722-0efcede062c9/shared-downloads' }}
+      SOURCE_MIRROR_URL: ${{ vars.SOURCE_MIRROR_URL || 'https://yocto-72c1c258-81bb-11ef-b722-0efcede062c9.s3.us-east-1.amazonaws.com/shared-downloads/' }}
 
     outputs:
       os_version: ${{ steps.balena-lib.outputs.os_version }}
@@ -462,6 +465,7 @@ jobs:
           BARYS_ARGUMENTS_VAR="${BARYS_ARGUMENTS_VAR} --bitbake-args --no-setscene"
           echo "BARYS_ARGUMENTS_VAR=${BARYS_ARGUMENTS_VAR}" >>"${GITHUB_ENV}"
 
+      # FIXME: Remove this when we shut down the NFS server in early 2025
       - name: Mount shared NFS cache
         if: vars.YOCTO_CACHE_HOST && contains(fromJSON(inputs.build-runs-on), 'self-hosted')
         continue-on-error: true
@@ -474,6 +478,7 @@ jobs:
           sudo chown -R "$(id -u):$(id -g)" "${MOUNTPOINT}"
           sudo mount -t nfs "${YOCTO_CACHE_HOST}:/" "${MOUNTPOINT}" -o fsc,nolock
 
+      # FIXME: Remove this when we shut down the NFS server in early 2025
       # https://wiki.yoctoproject.org/wiki/Enable_sstate_cache
       # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-MIRRORS
       # https://docs.yoctoproject.org/4.0.10/ref-manual/variables.html#term-PREMIRRORS
@@ -505,24 +510,30 @@ jobs:
           EOF
           cat "${AUTO_CONF_FILE}"
 
-      # # FIXME: We should probably change this to MIRRORS:append instead of PREMIRRORS:prepend
-      # # to avoid using our S3 egress as much as possible?
-      # # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass
-      # # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass
-      # # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before
-      # # attempting to fetch it from the upstream specified in SRC_URI within each recipe.
-      # - name: Add S3 shared-downloads to PREMIRRORS
-      #   env:
-      #     SOURCE_MIRROR_URL: https://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}.s3.${{ vars.AWS_REGION || 'us-east-1' }}.amazonaws.com/shared-downloads/
-      #   run: |
-      #     mkdir -p "$(dirname "${AUTO_CONF_FILE}")"
-      #     cat <<EOF >> "${AUTO_CONF_FILE}"
+      # https://docs.yoctoproject.org/4.0.10/ref-manual/classes.html?highlight=source_mirror#own-mirrors-bbclass
+      # https://github.com/openembedded/openembedded/blob/master/classes/own-mirrors.bbclass
+      # The own-mirrors class makes it easier to set up your own PREMIRRORS from which to first fetch source before
+      # attempting to fetch it from the upstream specified in SRC_URI within each recipe.
+      - name: Add S3 shared-downloads to MIRRORS
+        if: env.SOURCE_MIRROR_URL
+        env:
+          SOURCE_MIRROR_URL: ${{ env.SOURCE_MIRROR_URL }}
+        run: |
+          mkdir -p "$(dirname "${AUTO_CONF_FILE}")"
+          cat <<EOF >> "${AUTO_CONF_FILE}"
 
-      #     INHERIT += "own-mirrors"
-      #     SOURCE_MIRROR_URL = "${SOURCE_MIRROR_URL}"
+          MIRRORS:append = "\\
+            cvs://.*/.* ${SOURCE_MIRROR_URL} \\
+            svn://.*/.* ${SOURCE_MIRROR_URL} \\
+            git://.*/.* ${SOURCE_MIRROR_URL} \\
+            hg://.*/.* ${SOURCE_MIRROR_URL} \\
+            bzr://.*/.* ${SOURCE_MIRROR_URL} \\
+            https?$://.*/.* ${SOURCE_MIRROR_URL} \\
+            ftp://.*/.*  ${SOURCE_MIRROR_URL} \\
+          "
 
-      #     EOF
-      #     cat "${AUTO_CONF_FILE}"
+          EOF
+          cat "${AUTO_CONF_FILE}"
 
       # Use local S3 cache on self-hosted runners
       # https://github.com/tespkg/actions-cache
@@ -549,11 +560,8 @@ jobs:
           key: ${{ inputs.machine }}-sstate-${{ github.sha }}
           restore-keys: |
             ${{ inputs.machine }}-sstate-
-          # FIXME: Include the shared-downloads directory with the sstate cache for now until
-          # we fully switch to AWS S3 for shared-downloads
           path: |
             ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate
-            ${{ github.workspace }}/shared/shared-downloads
 
       # All preperation complete before this step
       # Start building balenaOS
@@ -619,11 +627,8 @@ jobs:
           region: local
           use-fallback: ${{ github.event.repository.private != true }}
           key: ${{ inputs.machine }}-sstate-${{ github.sha }}
-          # FIXME: Include the shared-downloads directory with the sstate cache for now until
-          # we fully switch to AWS S3 for shared-downloads
           path: |
             ${{ github.workspace }}/shared/${{ inputs.machine }}/sstate
-            ${{ github.workspace }}/shared/shared-downloads
 
       # https://github.com/unfor19/install-aws-cli-action
       - name: Setup awscli
@@ -639,29 +644,27 @@ jobs:
           # https://github.com/orgs/community/discussions/26636#discussioncomment-3252664
           mask-aws-account-id: false
 
-      # # Sync shared downloads to S3 to use as a sources mirror in case original sources are not available.
-      # # Exlude all directories and temp files as we only want the content and the .done files.
-      # # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html
-      # - name: Sync shared downloads to S3
-      #   # Do not publish shared downloads for pull_request_target events to prevent cache poisoning
-      #   # Do not publish shared downloads for private device-types as the mirror is public-read
-      #   if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false'
-      #   # Ignore errors for now, as we may have upload conflicts with other jobs
-      #   continue-on-error: true
-      #   env:
-      #     SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads
-      #     S3_ACL: public-read
-      #     S3_SSE: AES256
-      #     # FIXME: This should be a a new bucket used only for shared-downloads (one for staging, one for production)
-      #     S3_URL: "s3://${{ vars.AWS_S3_BUCKET || vars.S3_BUCKET }}/shared-downloads"
-      #     S3_REGION: ${{ vars.AWS_REGION || 'us-east-1' }}
-      #   # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks
-      #   # created in the build container runtime.
-      #   run: |
-      #     sudo ln -sf "${{ github.workspace }}" /work
-      #     ls -al "${SHARED_DOWNLOADS_DIR}/"
-      #     aws s3 sync --sse="${S3_SSE}" --acl="${S3_ACL}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" \
-      #       --exclude "*/*" --exclude "*.tmp" --size-only --follow-symlinks --no-progress
+      # Sync shared downloads to S3 to use as a sources mirror in case original sources are not available.
+      # Exlude all directories and temp files as we only want the content and the .done files.
+      # https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/sync.html
+      - name: Sync shared downloads to S3
+        # Do not publish shared downloads for pull_request_target events to prevent cache poisoning
+        # Do not publish shared downloads for private device-types as the mirror is public-read
+        if: github.event_name != 'pull_request_target' && steps.balena-lib.outputs.is_private == 'false' && env.SOURCE_MIRROR_S3_URL
+        # Ignore errors for now, as we may have upload conflicts with other jobs
+        continue-on-error: true
+        env:
+          SHARED_DOWNLOADS_DIR: ${{ github.workspace }}/shared/shared-downloads
+          S3_SSE: AES256
+          S3_URL: ${{ env.SOURCE_MIRROR_S3_URL }}
+          S3_REGION: ${{ env.SOURCE_MIRROR_REGION }}
+        # Create a symlink to the from the relative container path to the workspace in order to resolve symlinks
+        # created in the build container runtime.
+        run: |
+          sudo ln -sf "${{ github.workspace }}" /work
+          du -cksh "${SHARED_DOWNLOADS_DIR}/*"
+          aws s3 sync --sse="${S3_SSE}" "${SHARED_DOWNLOADS_DIR}/" "${S3_URL}/" \
+            --exclude "*/*" --exclude "*.tmp" --size-only --follow-symlinks --no-progress
 
       # TODO: pre-install on self-hosted-runners
       # Needed by the yocto job to zip artifacts - Don't remove