Merge branch 'lukas/gh/fix-prebuilt-cache-handling' into maint

* lukas/gh/fix-prebuilt-cache-handling: Update GitHub Action Versions gh: NO_CACHE if too many deleted gh: Pass args as env vars as DELETED can become very large gh: Rebuild bootstrap if any bootstrap apps changed gh: Fix argument bug when changing what has changed in prebuilt gh: Build correct docker image for cross, 32 and clang gh: Split otp_src and otp_cache into separate cache actions gh: Pin kerl version gh: Fix perf install gh: Don't use key for cache restore gh: Backport github actions to OTP-26
dgud · Oct 27, 2023 · 852cf26 · 852cf26
2 parents 15286ce + dd3e46c
commit 852cf26
Show file tree

Hide file tree

Showing 13 changed files with 237 additions and 134 deletions.
diff --git a/.github/actions/build-base-image/action.yaml b/.github/actions/build-base-image/action.yaml
@@ -43,16 +43,18 @@ runs:
         shell: bash
         run: .github/scripts/build-base-image.sh "${{ inputs.BASE_BRANCH }}" "${{ inputs.TYPE }}"
 
-      - name: Cache pre-built tar archives
+      - name: Cache pre-built src
         if: inputs.BUILD_IMAGE == 'true'
         uses: actions/cache@v3
         with:
-            path: |
-                otp_src.tar.gz
-                otp_cache.tar.gz
-            key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
-            restore-keys: |
-                prebuilt-${{ github.ref_name }}-${{ github.sha }}
+            path: otp_src.tar.gz
+            key: prebuilt-src-${{ github.ref_name }}-${{ github.sha }}
+      - name: Cache pre-built binaries
+        if: inputs.BUILD_IMAGE == 'true'
+        uses: actions/cache@v3
+        with:
+            path: otp_cache.tar.gz
+            key: prebuilt-cache-${{ inputs.TYPE }}-${{ github.ref_name }}-${{ github.sha }}
       - name: Build image
         if: inputs.BUILD_IMAGE == 'true'
         shell: bash -euxo pipefail {0}
@@ -61,5 +63,5 @@ runs:
           rm -f otp_{src,cache}.tar.gz
           docker build --tag otp \
             --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
-            --file ".github/dockerfiles/Dockerfile.64-bit" \
+            --file ".github/dockerfiles/Dockerfile.${{ inputs.TYPE }}" \
             .github/
diff --git a/.github/dockerfiles/Dockerfile.ubuntu-base b/.github/dockerfiles/Dockerfile.ubuntu-base
@@ -48,7 +48,7 @@ RUN mkdir /buildroot /tests /otp && chown ${USER}:${GROUP} /buildroot /tests /ot
 ## We install the latest version of the previous three releases in order to do
 ## backwards compatability testing of Erlang.
 RUN apt-get install -y git curl && \
-    curl -L https://raw.githubusercontent.com/kerl/kerl/master/kerl > /usr/bin/kerl && \
+    curl -L https://raw.githubusercontent.com/kerl/kerl/3.1.0/kerl > /usr/bin/kerl && \
     chmod +x /usr/bin/kerl && \
     kerl update releases && \
     LATEST=$(kerl list releases | tail -1 | awk -F '.' '{print $1}') && \
@@ -68,7 +68,7 @@ RUN apt-get install -y git curl && \
 ARG EXTRA_LIBS="erlang erlang-doc"
 RUN apt-get install -y \
     unixodbc odbc-postgresql postgresql ssh openssh-server groff-base gdb \
-    tinyproxy bind9 nsd expect vsftpd python emacs nano vim \
+    tinyproxy knot ldnsutils expect vsftpd python emacs nano vim \
     linux-tools-common linux-tools-generic jq \
     xvfb libgl1-mesa-dri && \
     for lib in ${EXTRA_LIBS}; do apt-get install -y ${lib}; done && \

diff --git a/.github/dockerfiles/init.sh b/.github/dockerfiles/init.sh
@@ -11,7 +11,7 @@ sudo /usr/sbin/sshd
 sudo service postgresql start
 
 sudo -E bash -c "apt-get update && apt-get install -y linux-tools-common linux-tools-generic"
-sudo -E bash -c "apt-get install -y linux-tools-$(uname-r)" || true
+sudo -E bash -c "apt-get install -y linux-tools-$(uname -r)" || true
 
 sudo bash -c "Xvfb :99 -ac -screen 0 1920x1080x24 -nolisten tcp" &
 export DISPLAY=:99

diff --git a/.github/scripts/build-base-image.sh b/.github/scripts/build-base-image.sh
@@ -12,6 +12,10 @@ esac
 
 if [ -z "${BASE_TAG}" ]; then
     BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.${2}" | head -1 | tr '=' ' ' | awk '{print $3}')
+    ## If this script is used on pre 25 releases
+    if [ -z "${BASE_TAG}" ]; then
+        BASE_TAG=$(grep "FROM " ".github/dockerfiles/Dockerfile.${2}" | head -1 | awk '{print $2}')
+    fi
 fi
 
 case "${BASE_TAG}" in

diff --git a/.github/scripts/restore-from-prebuilt.sh b/.github/scripts/restore-from-prebuilt.sh
@@ -1,13 +1,33 @@
 #!/bin/bash
+## restore-from-prebuilt.sh CACHE_SRC_DIR TARGET [ARCHIVE]
+##
+## This script attempts to restore as much as possible from a previous
+## CI run so that we don't have to build everything all the time.
+## It works by merging the contents of:
+##   * ${ARCHIVE} - The original source code, created by git archive
+##   * ${CACHE_SOURCE_DIR}/otp_src.tar.gz - The pre-built tar archive
+##   * ${CACHE_SOURCE_DIR}/otp_cache.tar.gz - A cache of many binary files
+##
+## otp_src and otp_cache can be either from a different step in the same CI run
+## or from a different CI run altogether.
+##
+## The archives above are then processed and placed into a new ${TARGET} archive.
+##
+## When running this script using the contents of a previous CI run you also have
+## to set the NO_CACHE, BOOTSTRAP, CONFIGURE, EVENT and DELETED environment
+## variables so that the correct parts of otp_src and otp_cache can be deleted.
+##
+##  * NO_CACHE - Don't use cache at all. Set if .github has changed or too many files have changed.
+##  * BOOTSTRAP - Don't cache any beam files. Set if bootstrap has changed.
+##  * CONFIGURE - Don't use any native cached. Set if configure has changed.
+##  * DELETED - Which files have been deleted and should therefore be deleted in the cache.
+##  * EVENT - The github event that triggered the change, currently unused.
 
 set -xe
 
 CACHE_SOURCE_DIR="$1"
 TARGET="$2"
 ARCHIVE="$3"
-EVENT="$4"
-DELETED="$5"
-CHANGES="$9"
 
 if [ ! -f "${CACHE_SOURCE_DIR}/otp_src.tar.gz" ] || [ "${NO_CACHE}" = "true" ]; then
     cp "${ARCHIVE}" "${TARGET}"
@@ -27,9 +47,12 @@ mkdir "${ARCHIVE_DIR}"
 echo "::group::{Restore cached files}"
 tar -C "${CACHE_DIR}/" -xzf "${CACHE_SOURCE_DIR}/otp_src.tar.gz"
 
-## If configure scripts have NOT changed, we can restore configure and other C/java programs
-if [ -z "${CONFIGURE}" ] || [ "${CONFIGURE}" = "false" ]; then
-    tar -C "${CACHE_DIR}/" -xzf "${CACHE_SOURCE_DIR}/otp_cache.tar.gz"
+## If we have a binary cache
+if [ -f "${CACHE_SOURCE_DIR}/otp_cache.tar.gz" ]; then
+    ## If configure scripts have NOT changed, we can restore configure and other C/java programs
+    if [ -z "${CONFIGURE}" ] || [ "${CONFIGURE}" = "false" ]; then
+        tar -C "${CACHE_DIR}/" -xzf "${CACHE_SOURCE_DIR}/otp_cache.tar.gz"
+    fi
 fi
 
 ## If bootstrap has been changed, we do not use the cached .beam files
@@ -71,29 +94,45 @@ if [ -n "${ARCHIVE}" ]; then
     ## Directory permissions in the archive and cache are for some reason different...
     chmod -R g-w "${ARCHIVE_DIR}/"
 
-    ## rlpgoD is the same as --archive, but without --times
-    RSYNC_ARGS=(-rlpgoD --itemize-changes --verbose --checksum --update "${EXCLUDE_BOOTSTRAP[@]}" "${ARCHIVE_DIR}/otp/" "${CACHE_DIR}/")
-
     CHANGES="${TMP_DIR}/changes"
     PREV_CHANGES="${TMP_DIR}/prev-changes"
 
     touch "${PREV_CHANGES}"
 
     ## Below follows some rules about when we do not want to use the cache
-    ## The rules are run multiple times so that if any rule triggeres a delte
+    ## The rules are run multiple times so that if any rule triggeres a delete
     ## we will re-run the rules again with the new changes.
     for i in $(seq 1 10); do
 
         echo "::group::{Run ${i} at pruning cache}"
 
-        ## First do a dry run to see if we need to delete anything from cache
+        ## rlpgoD is the same as --archive, but without --times
+        RSYNC_ARGS=(-rlpgoD --itemize-changes --verbose --checksum --update "${EXCLUDE_BOOTSTRAP[@]}" "${ARCHIVE_DIR}/otp/" "${CACHE_DIR}/")
+
+        ## We do a dry run to see if we need to purge anything from cache
         rsync --dry-run "${RSYNC_ARGS[@]}" | grep '^\(>\|c\)' > "${TMP_DIR}/changes"
         cat "${TMP_DIR}/changes"
 
+        ## If no new changes were done, we are done and can quit the loop
         if cmp -s "${CHANGES}" "${PREV_CHANGES}"; then
             break;
         fi
 
+        ### If any of the applications in the secondary or tertiary bootstrap have changed
+        ### we delete prebuilt.files which will trigger a rebuilt of the bootstrap
+        echo "::group::{Run ${i}: bootstrap applications}"
+        SECONDARY_BOOTSTRAP=(parsetools sasl asn1)
+        TERTIARY_BOOTSTRAP=(parsetools wx public_key erl_interface syntax_tools \
+                          snmp runtime_tools xmerl common_test)
+        for app in ${SECONDARY_BOOTSTRAP[@]} ${TERTIARY_BOOTSTRAP[@]}; do
+            if grep "lib/\(${app}\)" "${CHANGES}"; then
+                echo "Delete prebuilt.files and include bootstrap in sync" >&2
+                rm -f "${CACHE_DIR}/prebuilt.files"
+                EXCLUDE_BOOTSTRAP=()
+                break
+            fi
+        done
+
         ### If any parse transform is changed we recompile everything as we have
         ### no idea what it may change. If the parse transform calls any other
         ### modules we really should delete the cache for those as well, but

diff --git a/.github/scripts/sync-github-prs.es b/.github/scripts/sync-github-prs.es
@@ -5,9 +5,14 @@
 %% into the Target folder. It tries its best to not create too large
 %% files so that gh will still be happy with us when this is published to
 %% gh pages
+-module('sync-github-prs').
 -mode(compile).
 
 main([Repo, Target]) ->
+
+    io:format("Updating PRs in ~ts, current PRs are: ~p~n",
+              [Target, filelib:wildcard(filename:join(Target,"*"))]),
+
     AllOpenPrs = ghapi("gh api --paginate -X GET /repos/"++Repo++"/pulls -f state=open"),
     %% Download all updates, there really should not be any to download as they
     %% are updated when a PR is updated, but we do it anyways just to be safe.
@@ -26,7 +31,10 @@ main([Repo, Target]) ->
                   false ->
                       cmd("rm -rf " ++ filename:join(Target,PRNo))
               end
-      end, AllPrs);
+      end, AllPrs),
+
+    purge_prs(Target);
+
 main([Repo, Target, PRNo]) ->
     handle_prs(Repo, Target, [ghapi("gh api /repos/"++Repo++"/pulls/"++PRNo)]).
 
@@ -68,6 +76,9 @@ handle_pr(_Repo, Target,
             io:format("Checking for ~ts~n", [filename:join(PRDir, Ident)]),
             case file:read_file_info(filename:join(PRDir, Ident)) of
                 {error, enoent} ->
+                    io:format("Did not find ~ts. Files in dir are: ~p~n",
+                              [filename:join(PRDir, Ident),
+                               filelib:wildcard(filename:join(PRDir, "*"))]),
                     cmd("rm -rf "++PRDir),
                     ok = file:make_dir(PRDir),
                     ok = file:write_file(filename:join(PRDir,Ident), integer_to_list(Number)),
@@ -100,6 +111,10 @@ handle_pr(_Repo, Target,
                              ok = filelib:ensure_dir(CTLogsIndex),
                              ok = file:write_file(CTLogsIndex, ["No test logs found for ", Sha])
                     end,
+                    %% If we ever want to de-duplicate the docs, this command will create a
+                    %% stable md5sum.
+                    %% (cd $dir && find doc lib erts-* -type f \! -path "lib/jinterface-*" \! -name erlresolvelinks.js \! -name index.html \! -name release_notes.html \! -name users_guide.html \! -name internal_docs.html \! -name "*.eix" -exec md5sum {} \;) | sort -k 2 | awk "{print $1}" | md5sum
+                    %% where $dir is the pr directory.
                     DocIndex = filename:join([PRDir,"doc","index.html"]),
                     case file:read_file_info(DocIndex) of
                         {ok, _} -> ok;
@@ -154,6 +169,31 @@ purge_suite(SuiteFilePath) ->
               end, filelib:wildcard(filename:join(SuiteDir,"*.html")))
     end.
 
+%% If we have more the 10 GB of PR data we need to remove some otherwise
+%% github actions will not work them. So we purge the largest files until we
+%% reach the 10 GB limit.
+purge_prs(Target) ->
+    %% Start by deleting all data from common_test test runs as they are huge.
+    os:cmd("rm -rf "++Target++"*/ct_logs/ct_run*/*common_test_test*/run*/log_private/ct_run*"),
+    Files = string:split(cmd("find " ++ Target ++ " -type f -a "
+                             "-name \\! suite.log.html -exec du -a {} \\+"),"\n",all),
+    SortedFiles =
+        lists:sort(fun([A|_]=As,[B|_]=Bs) ->
+                               binary_to_integer(A) >= binary_to_integer(B)
+                   end, [string:split(F,"\t") || F <- Files, F =/= <<>>]),
+    purge_prs(SortedFiles, Target, get_directory_size(Target)).
+purge_prs(Files, Target, Size) when Size > 10_000_000_000 ->
+    {H,T} = lists:split(10, Files),
+    [file:write_file(File, io_lib:format("Large file (~p bytes) truncated", [Sz]))
+     || [Sz, File] <- H],
+    purge_prs(T, Target, get_directory_size(Target));
+purge_prs(_, _, _) ->
+    ok.
+
+get_directory_size(Dir) ->
+    binary_to_integer(hd(string:split(cmd("du -b --max-depth=0 " ++ Dir),"\t"))).
+
+
 ghapi(CMD) ->
     decode(cmd(CMD)).
 

diff --git a/.github/scripts/sync-github-releases.sh b/.github/scripts/sync-github-releases.sh
@@ -211,37 +211,40 @@ if [ ${UPLOADED} = true ]; then
 fi
 
 ## If no assets were uploaded, we try to build one instead
-if [ ${UPLOADED} = false ] && [ ${#MISSING_PREBUILD[0]} != 0 ]; then
-    name="${MISSING_PREBUILD[0]}"
-    stripped_name=$(_strip_name "${name}")
-    git clone https://github.com/erlang/otp -b "${name}" otp_src
-    if [ -f otp_src/.github/scripts/init-pre-release.sh ]; then
-        (cd otp_src && ERL_TOP=$(pwd) .github/scripts/init-pre-release.sh)
-    else
-        (cd otp_src && ERL_TOP=$(pwd) ../.github/scripts/init-pre-release.sh)
-    fi
-    case ${stripped_name} in
-        23.**)
-            ## The 32-bit dockerfile build the doc chunks which we want
-            ## to include in VSN >= 23.
-            docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
-                   -f otp_src/.github/dockerfiles/Dockerfile.32-bit .
-            ;;
-        *)
-            docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
-                   -f otp_src/.github/dockerfiles/Dockerfile.64-bit .
-            ;;
-    esac
-    docker run -v "$PWD":/github otp \
-           "/github/scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
-    .github/scripts/release-docs.sh
-    .github/scripts/create-artifacts.sh downloads "${name}"
-
-    ## Delete any artifacts that we should not upload
-    for artifact in dowloads/*; do
-        if ! echo "${RI[@]}" | grep "${artifact}" 2> /dev/null > /dev/null; then
-            rm -f "downloads/${artifact}"
+if [ ${UPLOADED} = false ]; then
+    for name in "${MISSING_PREBUILD[@]}"; do
+        stripped_name=$(_strip_name "${name}")
+        release=$(echo "${stripped_name}" | awk -F. '{print $1}')
+        if [[ $release < 24 ]]; then
+            ## Releases before 24 are no longer supported and are a bit different
+            ## from 24+ so I've removed support for them
+            echo "Skipping old release ${name}"
+            continue;
+        fi
+        echo "Building pre-build and docs for ${name}"
+        git clone https://github.com/erlang/otp -b "${name}" otp_src
+        if [ -f otp_src/.github/scripts/init-pre-release.sh ]; then
+            (cd otp_src && ERL_TOP=$(pwd) .github/scripts/init-pre-release.sh)
+        else
+            (cd otp_src && ERL_TOP=$(pwd) ../.github/scripts/init-pre-release.sh)
         fi
+        (cd otp_src && BASE_USE_CACHE=false GITHUB_OUTPUT=.tmp ../.github/scripts/build-base-image.sh maint-${release} 64-bit)
+        docker build -t otp --build-arg ARCHIVE=otp_src/otp_src.tar.gz \
+               -f otp_src/.github/dockerfiles/Dockerfile.64-bit .
+        docker run -v "$PWD":/github otp \
+               "/github/scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
+        .github/scripts/release-docs.sh
+        .github/scripts/create-artifacts.sh downloads "${name}"
+
+        ## Delete any artifacts that we should not upload
+        for artifact in dowloads/*; do
+            if ! echo "${RI[@]}" | grep "${artifact}" 2> /dev/null > /dev/null; then
+                rm -f "downloads/${artifact}"
+            fi
+        done
+        _upload_artifacts "${name}"
+
+        ## We only update one release per call to sync-github-releases
+        break
     done
-    _upload_artifacts "${name}"
 fi
diff --git a/.github/workflows/actions-updater.yaml b/.github/workflows/actions-updater.yaml
@@ -1,22 +1,29 @@
-name: GitHub Actions Updater
+name: GitHub Actions Version Updater
 
+# Controls when the action will run.
 on:
+  workflow_dispatch:
   schedule:
-    # Automatically run on the 1st of every month
-    - cron:  '0 0 1 * *'
+    # Automatically run on every Sunday
+    - cron:  '0 0 * * 0'
 
 jobs:
   build:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/[email protected]
+      - name: Generate token
+        id: generate_token
+        uses: actions/[email protected]
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PEM }}
+
+      - uses: actions/[email protected]
+        with:
+          token: ${{ steps.generate_token.outputs.token }}
 
       - name: Run GitHub Actions Version Updater
-        uses: saadmk11/github-actions-version-updater@v0.7.3
+        uses: saadmk11/github-actions-version-updater@v0.8.1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          commit_message: "Updating GitHub actions to their latest versions"
-          pull_request_labels: "team:IS"
+          token: ${{ steps.generate_token.outputs.token }}