From 80c9e938a8f925990cb592beb70809dc7f947529 Mon Sep 17 00:00:00 2001 From: Lucas Harms Date: Tue, 7 Dec 2021 16:21:34 -0600 Subject: [PATCH] Separate arm64 workflow --- .github/workflows/build-and-release.yaml | 30 +++-- .github/workflows/build-arm64.yaml | 146 +++++++++++++++++++++++ 2 files changed, 160 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/build-arm64.yaml diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml index ffaea393d..7ea821d2c 100644 --- a/.github/workflows/build-and-release.yaml +++ b/.github/workflows/build-and-release.yaml @@ -25,7 +25,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v2 with: - python-version: '3.7' + python-version: '3.8' architecture: 'x64' - name: Install dependencies @@ -43,7 +43,7 @@ jobs: run: flake8 - build-amd64: + build: runs-on: ubuntu-20.04 strategy: @@ -75,11 +75,14 @@ jobs: python_version: "3.9" run_tests: true - - arch: arm64 - build_name: py2.7 - dockerhub_push: true - python_version: "2.7" - run_tests: false + # The qemu arm64 builds are *very* slow so they are split out into + # their own workflow + + # - arch: arm64 + # build_name: py2.7 + # dockerhub_push: true + # python_version: "2.7" + # run_tests: false # - arch: arm64 # build_name: py3.8 # dockerhub_push: true @@ -159,7 +162,7 @@ jobs: multi-arch-manifest: needs: - - build-amd64 + - build runs-on: ubuntu-20.04 @@ -167,7 +170,7 @@ jobs: matrix: include: - build_name: py2.7 - # - build_name: py3.8 + - build_name: py3.8 steps: - name: Get short sha @@ -190,14 +193,9 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - # - name: Sleep 30s - # run: | - # echo "sleep 30s for Docker Hub to update" - # sleep 30 - - name: Create manifest run: | docker buildx imagetools create \ -t ${{ env.DOCKER_IMAGE_TAG }} \ - ${{ env.DOCKER_IMAGE_TAG }}-amd64 \ - ${{ env.DOCKER_IMAGE_TAG }}-arm64 + ${{ env.DOCKER_IMAGE_TAG }}-amd64 + # ${{ env.DOCKER_IMAGE_TAG }}-arm64 diff --git a/.github/workflows/build-arm64.yaml b/.github/workflows/build-arm64.yaml new file mode 100644 index 000000000..aee1e44df --- /dev/null +++ b/.github/workflows/build-arm64.yaml @@ -0,0 +1,146 @@ +name: Build and Release + +on: + push: + branches: + - '*' + +concurrency: + group: build-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + DOCKERHUB_NAMESPACE: closeio + PROJECT: sync-engine + # TODO: Remove this after migration + TAG_SUFFIX: "-gha" + +jobs: + build-arm64: + runs-on: ubuntu-20.04 + + strategy: + matrix: + include: + - arch: arm64 + build_name: py2.7 + dockerhub_push: true + python_version: "2.7" + run_tests: false + - arch: arm64 + build_name: py3.8 + dockerhub_push: true + python_version: "3.8" + run_tests: false + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Get short sha + id: vars + run: | + echo ::set-output name=sha_short::${GITHUB_SHA::16} + shell: bash + + - name: Prepare environment + run: | + if [ "$(git log -1 --pretty=%B | head -n 1 | grep '#notests')" ]; then + echo 'SKIP_TESTS="1"' >> $GITHUB_ENV + fi + echo "DOCKER_IMAGE_TAG=${{ env.DOCKERHUB_NAMESPACE }}/${{ env.PROJECT }}:${{ steps.vars.outputs.sha_short }}-${{ matrix.build_name}}${{ env.TAG_SUFFIX }}-${{ matrix.arch }}" >> $GITHUB_ENV + shell: bash + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker context for Buildx + id: buildx-context + run: | + docker context create container-builder + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + version: latest + endpoint: container-builder + + - name: Login to DockerHub + # if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }} + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Build and push sync-engine images + uses: docker/build-push-action@v2 + with: + platforms: linux/${{ matrix.arch }} + push: false + load: true + build-args: + PYTHON_VERSION=${{ matrix.python_version }} + tags: | + ${{ env.DOCKER_IMAGE_TAG }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Run tests + if: ${{ matrix.run_tests }} + run: | + docker tag ${{ env.DOCKER_IMAGE_TAG }} ${{ env.PROJECT }}_app + if [[ -z $SKIP_TESTS ]]; then + docker-compose run app bash -ec ' + bin/wait-for-it.sh mysql:3306 \ + && NYLAS_ENV=test pytest --cov-report= --cov=inbox tests/ \ + && coverage html -d pythoncov + ' + else + echo Skipping tests + fi + + - name: Push image + if: ${{ matrix.dockerhub_push }} + run: | + docker push ${{ env.DOCKER_IMAGE_TAG }} + + multi-arch-manifest: + needs: + - build-arm64 + + runs-on: ubuntu-20.04 + + strategy: + matrix: + include: + - build_name: py2.7 + - build_name: py3.8 + + steps: + - name: Get short sha + id: vars + run: | + echo ::set-output name=sha_short::${GITHUB_SHA::16} + shell: bash + + - name: Prepare environment + run: | + if [ "$(git log -1 --pretty=%B | head -n 1 | grep '#notests')" ]; then + echo 'SKIP_TESTS="1"' >> $GITHUB_ENV + fi + echo "DOCKER_IMAGE_TAG=${{ env.DOCKERHUB_NAMESPACE }}/${{ env.PROJECT }}:${{ steps.vars.outputs.sha_short }}-${{ matrix.build_name}}${{ env.TAG_SUFFIX }}" >> $GITHUB_ENV + shell: bash + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + # Because the arm64 builds are so much slower, we assume they will just + # be appended to the existing manifest created by `build-and-release.yaml`. + - name: Create manifest + run: | + docker buildx imagetools create --append \ + -t ${{ env.DOCKER_IMAGE_TAG }} \ + ${{ env.DOCKER_IMAGE_TAG }}-arm64 \ No newline at end of file