From e6f5a6a4c22d8949d1fd3eb9c8e3a651d38ef6f7 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Tue, 1 Aug 2023 13:30:18 -0700 Subject: [PATCH 1/7] chore(ci): support additional paths in the build cache key --- .github/actions/build-cache-key/action.yml | 86 +++++++++++++++++----- .github/workflows/release.yml | 2 + 2 files changed, 71 insertions(+), 17 deletions(-) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 7b91acb1210..65a640b07e5 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -8,6 +8,9 @@ inputs: description: 'String prefix applied to the build cache key' required: false default: 'build' + paths: + description: 'Additional paths (newline-delimited) to use in cache key generation' + required: false outputs: cache-key: @@ -20,26 +23,75 @@ runs: - name: Generate cache key id: cache-key shell: bash + env: + EXTRA_PATHS: ${{ inputs.paths }} run: | + echo "::group::Enumerate Paths" + + LIST=$(mktemp) + + add_file() { + local f=$1 + echo "path spec: $f" + + git ls-files \ + --full-name \ + --recurse-submodules \ + "$f" \ + >> "$LIST" + } + # please keep these sorted - FILE_HASHES=( - ${{ hashFiles('.bazelignore') }} - ${{ hashFiles('.bazelrc') }} - ${{ hashFiles('.bazelversion') }} - ${{ hashFiles('.github/actions/build-cache-key/**') }} - ${{ hashFiles('.github/workflows/build.yml') }} - ${{ hashFiles('.requirements') }} - ${{ hashFiles('BUILD.bazel') }} - ${{ hashFiles('WORKSPACE') }} - ${{ hashFiles('bin/kong') }} - ${{ hashFiles('bin/kong-health') }} - ${{ hashFiles('build/**') }} - ${{ hashFiles('kong-*.rockspec') }} - ${{ hashFiles('kong.conf.default') }} - ${{ hashFiles('kong/**') }} + DEFAULT_PATHS=( + '.bazelignore' + '.bazelrc' + '.bazelversion' + '.github/actions/build-cache-key/**' + '.github/workflows/build.yml' + '.requirements' + 'BUILD.bazel' + 'WORKSPACE' + 'bin/kong' + 'bin/kong-health' + 'build/**' + 'kong-*.rockspec' + 'kong.conf.default' ) - HASH=$(sha256sum - <<< "${FILE_HASHES[*]}" | awk '{print $1}' ) - CACHE_KEY=${{ inputs.prefix }}::${HASH} + for fname in "${DEFAULT_PATHS[@]}"; do + add_file "$fname" + done + + if [[ -n ${EXTRA_PATHS:-} ]]; then + readarray -t EXTRA <<< "$EXTRA_PATHS" + + for fname in "${EXTRA[@]}"; do + if [[ -z ${fname:-} ]]; then + continue + fi + + add_file "$fname" + done + fi + + echo "::endgroup::" + + echo "::group::Hash Files" + + HASH=$(mktemp) + + sort \ + --stable \ + --unique \ + < "$LIST" \ + | while read -r fname; do + sha256sum "$fname" | tee -a "$HASH" + done + + echo "::endgroup::" + + ALL=$(sha256sum "$HASH" | awk '{print $1}') + + CACHE_KEY=${{ inputs.prefix }}::${ALL} echo "cache-key: ${CACHE_KEY}" echo "CACHE_KEY=${CACHE_KEY}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 29573dc9de3..5ac2365d1d6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -179,6 +179,8 @@ jobs: uses: ./.github/actions/build-cache-key with: prefix: ${{ matrix.label }}-build + paths: | + kong/** - name: Cache Packages id: cache-deps From d2e84bead045a49113a2048e0cfe9c300f1dba78 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 24 Aug 2023 11:19:50 -0700 Subject: [PATCH 2/7] caller passes in hashes directly --- .github/actions/build-cache-key/action.yml | 94 +++++++--------------- .github/workflows/release.yml | 4 +- 2 files changed, 33 insertions(+), 65 deletions(-) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 65a640b07e5..5a1101ef559 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -8,8 +8,8 @@ inputs: description: 'String prefix applied to the build cache key' required: false default: 'build' - paths: - description: 'Additional paths (newline-delimited) to use in cache key generation' + extra: + description: 'Additional values/file hashes to use in the cache key' required: false outputs: @@ -24,74 +24,42 @@ runs: id: cache-key shell: bash env: - EXTRA_PATHS: ${{ inputs.paths }} + EXTRA: ${{ inputs.extra }} run: | - echo "::group::Enumerate Paths" - - LIST=$(mktemp) - - add_file() { - local f=$1 - echo "path spec: $f" - - git ls-files \ - --full-name \ - --recurse-submodules \ - "$f" \ - >> "$LIST" - } - # please keep these sorted - DEFAULT_PATHS=( - '.bazelignore' - '.bazelrc' - '.bazelversion' - '.github/actions/build-cache-key/**' - '.github/workflows/build.yml' - '.requirements' - 'BUILD.bazel' - 'WORKSPACE' - 'bin/kong' - 'bin/kong-health' - 'build/**' - 'kong-*.rockspec' - 'kong.conf.default' + FILE_HASHES=( + ${{ hashFiles('.bazelignore') }} + ${{ hashFiles('.bazelrc') }} + ${{ hashFiles('.bazelversion') }} + ${{ hashFiles('.github/actions/build-cache-key/**') }} + ${{ hashFiles('.github/workflows/build.yml') }} + ${{ hashFiles('.requirements') }} + ${{ hashFiles('BUILD.bazel') }} + ${{ hashFiles('WORKSPACE') }} + ${{ hashFiles('bin/kong') }} + ${{ hashFiles('bin/kong-health') }} + ${{ hashFiles('build/**') }} + ${{ hashFiles('kong-*.rockspec') }} + ${{ hashFiles('kong.conf.default') }} ) - for fname in "${DEFAULT_PATHS[@]}"; do - add_file "$fname" - done - - if [[ -n ${EXTRA_PATHS:-} ]]; then - readarray -t EXTRA <<< "$EXTRA_PATHS" - - for fname in "${EXTRA[@]}"; do - if [[ -z ${fname:-} ]]; then - continue - fi + if [[ -n ${EXTRA:-} ]]; then + OFFSET=${#FILE_HASHES[@]} - add_file "$fname" - done + readarray \ + -O "$OFFSET" \ + -t \ + FILE_HASHES \ + <<< "$EXTRA" fi - echo "::endgroup::" - - echo "::group::Hash Files" - - HASH=$(mktemp) - - sort \ - --stable \ - --unique \ - < "$LIST" \ - | while read -r fname; do - sha256sum "$fname" | tee -a "$HASH" - done - - echo "::endgroup::" - - ALL=$(sha256sum "$HASH" | awk '{print $1}') + NORMALIZED=$( + printf '%s\n' "${FILE_HASHES[@]}" \ + | grep -vE '^$' \ + | sort --stable --unique + ) - CACHE_KEY=${{ inputs.prefix }}::${ALL} + HASH=$(sha256sum - <<< "$NORMALIZED" | awk '{print $1}' ) + CACHE_KEY=${{ inputs.prefix }}::${HASH} echo "cache-key: ${CACHE_KEY}" echo "CACHE_KEY=${CACHE_KEY}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ac2365d1d6..78e232d9242 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -179,8 +179,8 @@ jobs: uses: ./.github/actions/build-cache-key with: prefix: ${{ matrix.label }}-build - paths: | - kong/** + extra: | + ${{ hashFiles('kong/**') }} - name: Cache Packages id: cache-deps From 308c05e771c34fd782c263513c954fb018451359 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 24 Aug 2023 11:37:37 -0700 Subject: [PATCH 3/7] debug output --- .github/actions/build-cache-key/action.yml | 12 ++++++++++++ .github/workflows/build.yml | 5 +++++ 2 files changed, 17 insertions(+) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 5a1101ef559..55ee4f10459 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -43,14 +43,23 @@ runs: ${{ hashFiles('kong.conf.default') }} ) + echo "FILE_HASHES (init):" + printf '%q\n' "${FILE_HASHES[@]}" + if [[ -n ${EXTRA:-} ]]; then OFFSET=${#FILE_HASHES[@]} + echo "EXTRA:" + printf '%q\n' "$EXTRA" + readarray \ -O "$OFFSET" \ -t \ FILE_HASHES \ <<< "$EXTRA" + + echo "FILE_HASHES (extended):" + printf '%q\n' "${FILE_HASHES[@]}" fi NORMALIZED=$( @@ -59,6 +68,9 @@ runs: | sort --stable --unique ) + echo "FILE_HASHES (normalized):" + echo "$NORMALIZED" + HASH=$(sha256sum - <<< "$NORMALIZED" | awk '{print $1}' ) CACHE_KEY=${{ inputs.prefix }}::${HASH} echo "cache-key: ${CACHE_KEY}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 54ed720cd54..d01935fb0de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,11 @@ jobs: - name: Generate cache key id: cache-key uses: ./.github/actions/build-cache-key + with: + extra: | + ${{ hashFiles('kong/**') }} + ${{ hashFiles('kong/**') }} + ${{ hashFiles('kong*.rockspec') }} - name: Lookup build cache id: cache-deps From 9ff21c2813349a4d92452ebea69065be3e5b5eba Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 24 Aug 2023 11:43:36 -0700 Subject: [PATCH 4/7] s/printf/echo/ --- .github/actions/build-cache-key/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 55ee4f10459..32787ef637b 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -50,7 +50,7 @@ runs: OFFSET=${#FILE_HASHES[@]} echo "EXTRA:" - printf '%q\n' "$EXTRA" + echo "$EXTRA" readarray \ -O "$OFFSET" \ From d9f6288296f5c929787e93258db873ca88d658ce Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Thu, 24 Aug 2023 11:47:04 -0700 Subject: [PATCH 5/7] remove extra from build cache key --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d01935fb0de..54ed720cd54 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,11 +34,6 @@ jobs: - name: Generate cache key id: cache-key uses: ./.github/actions/build-cache-key - with: - extra: | - ${{ hashFiles('kong/**') }} - ${{ hashFiles('kong/**') }} - ${{ hashFiles('kong*.rockspec') }} - name: Lookup build cache id: cache-deps From 73e9df02257ba9fd408cd6003aefd01b093be1c8 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Wed, 6 Sep 2023 10:39:28 -0700 Subject: [PATCH 6/7] condense bash --- .github/actions/build-cache-key/action.yml | 31 ++++++---------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 32787ef637b..313cb675c1b 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -24,6 +24,7 @@ runs: id: cache-key shell: bash env: + PREFIX: ${{ inputs.prefix }} EXTRA: ${{ inputs.extra }} run: | # please keep these sorted @@ -43,35 +44,19 @@ runs: ${{ hashFiles('kong.conf.default') }} ) - echo "FILE_HASHES (init):" - printf '%q\n' "${FILE_HASHES[@]}" - if [[ -n ${EXTRA:-} ]]; then - OFFSET=${#FILE_HASHES[@]} - - echo "EXTRA:" - echo "$EXTRA" - readarray \ - -O "$OFFSET" \ + -O "${#FILE_HASHES[@]}" \ -t \ FILE_HASHES \ <<< "$EXTRA" - - echo "FILE_HASHES (extended):" - printf '%q\n' "${FILE_HASHES[@]}" fi - NORMALIZED=$( - printf '%s\n' "${FILE_HASHES[@]}" \ - | grep -vE '^$' \ - | sort --stable --unique + HASH=$(printf '%s\n' "${FILE_HASHES[@]}" \ + | grep -vE '^$' \ + | sort --stable --unique \ + | sha256sum - \ + | awk '{print $1}' ) - echo "FILE_HASHES (normalized):" - echo "$NORMALIZED" - - HASH=$(sha256sum - <<< "$NORMALIZED" | awk '{print $1}' ) - CACHE_KEY=${{ inputs.prefix }}::${HASH} - echo "cache-key: ${CACHE_KEY}" - echo "CACHE_KEY=${CACHE_KEY}" >> $GITHUB_OUTPUT + echo "CACHE_KEY=${PREFIX}::${HASH}" >> $GITHUB_OUTPUT From 0f1d89acb5687980e8923c82a12f150f79828d22 Mon Sep 17 00:00:00 2001 From: Michael Martin Date: Wed, 6 Sep 2023 10:41:51 -0700 Subject: [PATCH 7/7] print cache key too --- .github/actions/build-cache-key/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/build-cache-key/action.yml b/.github/actions/build-cache-key/action.yml index 313cb675c1b..3edde92a342 100644 --- a/.github/actions/build-cache-key/action.yml +++ b/.github/actions/build-cache-key/action.yml @@ -59,4 +59,4 @@ runs: | awk '{print $1}' ) - echo "CACHE_KEY=${PREFIX}::${HASH}" >> $GITHUB_OUTPUT + echo "CACHE_KEY=${PREFIX}::${HASH}" | tee -a $GITHUB_OUTPUT