Skip to content

Commit

Permalink
Merge branch 'lukas/gh/refactor-build-base-image' into maint
Browse files Browse the repository at this point in the history
* lukas/gh/refactor-build-base-image:
  gh: Update cached branch to 26
  gh: Refactor build-base-image to composite image
  • Loading branch information
garazdawi committed Sep 29, 2023
2 parents ef3ed9f + 4986a78 commit e022d28
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 185 deletions.
65 changes: 65 additions & 0 deletions .github/actions/build-base-image/action.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
name: Build Base Image
description: 'Builds the base testing image (using cache if possible)'

inputs:
BASE_BRANCH:
required: true
TYPE:
default: '64-bit'
BUILD_IMAGE:
default: true
github_token:
description: 'GITHUB_TOKEN'
default: '${{ github.token }}'

runs:
using: composite
steps:
- name: Cleanup GH Runner
shell: bash -euxo pipefail {0}
run: |
## Delete large files from runner to get more disk space
## See https://github.com/actions/runner-images/issues/2840
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Cache BASE image
uses: actions/cache@v3
if: inputs.TYPE == '64-bit' || inputs.TYPE == 'clang'
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}-${{ hashFiles('OTP_VERSION') }}

- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ inputs.github_token }}

- name: Build BASE image
shell: bash
run: .github/scripts/build-base-image.sh "${{ inputs.BASE_BRANCH }}" "${{ inputs.TYPE }}"

- name: Cache pre-built tar archives
if: inputs.BUILD_IMAGE == 'true'
uses: actions/cache@v3
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
if: inputs.BUILD_IMAGE == 'true'
shell: bash -euxo pipefail {0}
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
202 changes: 18 additions & 184 deletions .github/workflows/main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ jobs:
all: ${{ steps.apps.outputs.all }}
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/build-base-image
with:
BASE_BRANCH: ${{ env.BASE_BRANCH }}
BUILD_IMAGE: false
- name: Get applications
id: apps
run: |
Expand All @@ -52,19 +56,6 @@ jobs:
with:
name: otp_git_archive
path: otp_archive.tar.gz
- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}-${{ hashFiles('OTP-VERSION') }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
- name: Cache pre-built tar archives
id: pre-built-cache
uses: actions/cache@v3
Expand Down Expand Up @@ -324,37 +315,9 @@ jobs:

steps:
- uses: actions/checkout@v3
## Download docker images
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Docker login
uses: docker/login-action@v2
- uses: ./.github/actions/build-base-image
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
- name: Cache pre-built tar archives
uses: actions/cache@v3
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
BASE_BRANCH: ${{ env.BASE_BRANCH }}
- name: Build Erlang/OTP flavors and types
run: |
TYPES="opt debug lcnt"
Expand All @@ -379,32 +342,14 @@ jobs:

steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/build-base-image
with:
BASE_BRANCH: ${{ env.BASE_BRANCH }}
TYPE: ${{ matrix.type }}
- name: Download source archive
uses: actions/download-artifact@v3
with:
name: otp_prebuilt
- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Cleanup GH Runner
run: |
## Delete large files from runner to get more disk space
## See https://github.com/actions/runner-images/issues/2840
sudo rm -rf /usr/share/dotnet
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
- name: Cache BASE image
uses: actions/cache@v3
if: matrix.type == 'clang'
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Build base image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" ${{ matrix.type }}
- name: Build ${{ matrix.type }} image
run: |
mv otp_src.tar.gz .github/otp.tar.gz
Expand All @@ -417,37 +362,9 @@ jobs:
needs: pack
steps:
- uses: actions/checkout@v3
## Download docker images
- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
- name: Cache pre-built tar archives
uses: actions/cache@v3
- uses: ./.github/actions/build-base-image
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
BASE_BRANCH: ${{ env.BASE_BRANCH }}

## Build all the documentation
- name: Build documentation
Expand Down Expand Up @@ -487,37 +404,9 @@ jobs:
needs: pack
steps:
- uses: actions/checkout@v3
## Download docker images
- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
- name: Cache pre-built tar archives
uses: actions/cache@v3
- uses: ./.github/actions/build-base-image
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
BASE_BRANCH: ${{ env.BASE_BRANCH }}
- name: Install clang-format
run: |
docker build -t otp - <<EOF
Expand All @@ -544,37 +433,9 @@ jobs:
fail-fast: false
steps:
- uses: actions/checkout@v3
## Download docker images
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Docker login
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
- name: Cache pre-built tar archives
uses: actions/cache@v3
- uses: ./.github/actions/build-base-image
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
BASE_BRANCH: ${{ env.BASE_BRANCH }}
- name: Run tests
id: run-tests
run: |
Expand Down Expand Up @@ -621,38 +482,11 @@ jobs:
needs: test
steps:
- uses: actions/checkout@v3
- name: Cache BASE image
uses: actions/cache@v3
with:
path: otp_docker_base.tar
key: ${{ runner.os }}-${{ hashFiles('.github/dockerfiles/Dockerfile.ubuntu-base', '.github/scripts/build-base-image.sh') }}
- name: Docker login
uses: docker/login-action@v2
- uses: ./.github/actions/build-base-image
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build BASE image
run: .github/scripts/build-base-image.sh "${BASE_BRANCH}" 64-bit
BASE_BRANCH: ${{ env.BASE_BRANCH }}
- name: Download test results
uses: actions/download-artifact@v3
- name: Cache pre-built tar archives
uses: actions/cache@v3
with:
path: |
otp_src.tar.gz
otp_cache.tar.gz
key: prebuilt-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
restore-keys: |
prebuilt-${{ github.ref_name }}-${{ github.sha }}
- name: Build image
run: |
.github/scripts/restore-from-prebuilt.sh `pwd` .github/otp.tar.gz
rm -f otp_{src,cache}.tar.gz
docker build --tag otp \
--build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
--file ".github/dockerfiles/Dockerfile.64-bit" \
.github/
- name: Merge test results
run: |
shopt -s nullglob
Expand Down
3 changes: 2 additions & 1 deletion .github/workflows/update-base.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ jobs:
strategy:
matrix:
type: [debian-base,ubuntu-base,i386-debian-base]
branch: [master, maint, maint-25]
branch: [master, maint, maint-26]
fail-fast: false

steps:
- uses: actions/checkout@v3
Expand Down

0 comments on commit e022d28

Please sign in to comment.