diff --git a/.github/workflows/actions-updater.yaml b/.github/workflows/actions-updater.yaml
index 2a4980b3a40c..5bec7a747c5f 100644
--- a/.github/workflows/actions-updater.yaml
+++ b/.github/workflows/actions-updater.yaml
@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: actions/create-github-app-token@v1.5.0
+        uses: actions/create-github-app-token@v1.5.1
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PEM }}
diff --git a/.github/workflows/add-to-project.yaml b/.github/workflows/add-to-project.yaml
index 3539790c0eb5..874ef8724079 100644
--- a/.github/workflows/add-to-project.yaml
+++ b/.github/workflows/add-to-project.yaml
@@ -16,7 +16,7 @@ jobs:
     steps:
       - name: Generate token
         id: generate_token
-        uses: actions/create-github-app-token@v1.5.0
+        uses: actions/create-github-app-token@v1.5.1
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PEM }}
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index e4ed1ed7000d..8ea1b13e68ac 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -124,8 +124,14 @@ jobs:
             .github/
       - name: Build pre-built tar archives
         run: |
+          ## Do a little dance to add new files to the cache
+          gunzip otp_src.tar.gz
+          gunzip otp_archive.tar.gz
+          tar -f otp_src_archive.tar --concatenate otp_archive.tar otp_src.tar
+          ## The resulting otp_src.tar.gz needs to be opened with -i, which build-otp-tar does
+          gzip otp_src_archive.tar
           docker run -v $PWD:/github --entrypoint "" otp \
-            scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /github/otp_archive.tar.gz
+            scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /github/otp_src_archive.tar.gz
       - name: Build cache
         run: |
           if [ -f otp_cache.tar.gz ]; then
@@ -137,6 +143,12 @@ jobs:
           docker run -v $PWD:/github --entrypoint "" otp \
             bash -c 'set -x; C_APPS=$(ls -d ./lib/*/c_src); find Makefile ./make ./erts ./bin/`erts/autoconf/config.guess` ./lib/erl_interface ./lib/jinterface ${C_APPS} `echo "${C_APPS}" | sed -e 's:c_src$:priv:'` -type f -newer README.md \! -name "*.beam" \! -path "*/doc/*" | xargs tar --transform "s:^./:otp/:" -uvf /github/otp_cache.tar'
           gzip otp_cache.tar
+      - name: Upload concatenated archive
+        uses: actions/upload-artifact@v3.1.3
+        if: runner.debug == 1
+        with:
+          name: otp_src_archive
+          path: otp_src_archive.tar.gz
       - name: Upload pre-built tar archives
         uses: actions/upload-artifact@v3.1.3
         with:
diff --git a/scripts/build-otp-tar b/scripts/build-otp-tar
index 6fb251f6e66c..e03e26cc1b0e 100755
--- a/scripts/build-otp-tar
+++ b/scripts/build-otp-tar
@@ -439,7 +439,7 @@ else
 
     progress "Unpacking OTP source code into build-directory"
 
-    $gtar -z -x -C $build_root -f $src_tgz
+    $gtar -i -z -x -C $build_root -f $src_tgz
     if [ $? -ne 0 ]; then
 	error "Failed to unpack source"
     fi
@@ -530,7 +530,7 @@ if [ $? -ne 0 ]; then
 fi
 
 progress "Unpacking OTP source code into source-directory"
-$gtar -z -x -C $src_root -f $src_tgz
+$gtar -i -z -x -C $src_root -f $src_tgz
 if [ $? -ne 0 ]; then
     error "Failed to unpack source"
 fi