Separate arm64 workflow

closeio · Jan 29, 2024 · 80c9e93 · 80c9e93
1 parent 44fd7de
commit 80c9e93
Show file tree

Hide file tree

Showing 2 changed files with 160 additions and 16 deletions.
diff --git a/.github/workflows/build-and-release.yaml b/.github/workflows/build-and-release.yaml
@@ -25,7 +25,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
-          python-version: '3.7'
+          python-version: '3.8'
           architecture: 'x64'
 
       - name: Install dependencies
@@ -43,7 +43,7 @@ jobs:
         run: flake8
 
 
-  build-amd64:
+  build:
     runs-on: ubuntu-20.04
 
     strategy:
@@ -75,11 +75,14 @@ jobs:
             python_version: "3.9"
             run_tests: true
 
-          - arch: arm64
-            build_name: py2.7
-            dockerhub_push: true
-            python_version: "2.7"
-            run_tests: false
+          # The qemu arm64 builds are *very* slow so they are split out into 
+          # their own workflow
+
+          # - arch: arm64
+          #   build_name: py2.7
+          #   dockerhub_push: true
+          #   python_version: "2.7"
+          #   run_tests: false
           # - arch: arm64
           #   build_name: py3.8
           #   dockerhub_push: true
@@ -159,15 +162,15 @@ jobs:
 
   multi-arch-manifest:
     needs:
-      - build-amd64
+      - build
 
     runs-on: ubuntu-20.04
 
     strategy:
       matrix:
         include:
           - build_name: py2.7
-          # - build_name: py3.8
+          - build_name: py3.8
 
     steps:
       - name: Get short sha
@@ -190,14 +193,9 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
-      # - name: Sleep 30s
-      #   run: |
-      #     echo "sleep 30s for Docker Hub to update"
-      #     sleep 30
-
       - name: Create manifest
         run: |
           docker buildx imagetools create \
             -t ${{ env.DOCKER_IMAGE_TAG }} \
-            ${{ env.DOCKER_IMAGE_TAG }}-amd64 \
-            ${{ env.DOCKER_IMAGE_TAG }}-arm64
+            ${{ env.DOCKER_IMAGE_TAG }}-amd64
+            # ${{ env.DOCKER_IMAGE_TAG }}-arm64
diff --git a/.github/workflows/build-arm64.yaml b/.github/workflows/build-arm64.yaml
@@ -0,0 +1,146 @@
+name: Build and Release
+
+on:
+  push:
+    branches:
+      - '*'
+
+concurrency:
+  group: build-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  DOCKERHUB_NAMESPACE: closeio
+  PROJECT: sync-engine
+  # TODO: Remove this after migration
+  TAG_SUFFIX: "-gha"
+
+jobs:
+  build-arm64:
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        include:
+          - arch: arm64
+            build_name: py2.7
+            dockerhub_push: true
+            python_version: "2.7"
+            run_tests: false
+          - arch: arm64
+            build_name: py3.8
+            dockerhub_push: true
+            python_version: "3.8"
+            run_tests: false
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Get short sha
+        id: vars
+        run: |
+          echo ::set-output name=sha_short::${GITHUB_SHA::16}
+        shell: bash
+
+      - name: Prepare environment
+        run: |
+          if [ "$(git log -1 --pretty=%B | head -n 1 | grep '#notests')" ]; then
+            echo 'SKIP_TESTS="1"' >> $GITHUB_ENV
+          fi
+          echo "DOCKER_IMAGE_TAG=${{ env.DOCKERHUB_NAMESPACE }}/${{ env.PROJECT }}:${{ steps.vars.outputs.sha_short }}-${{ matrix.build_name}}${{ env.TAG_SUFFIX }}-${{ matrix.arch }}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+
+      - name: Set up Docker context for Buildx
+        id: buildx-context
+        run: |
+          docker context create container-builder
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+        with:
+          version: latest
+          endpoint: container-builder
+
+      - name: Login to DockerHub
+        # if: ${{ github.ref == 'master' && github.event.pull_request.merged == true }}
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push sync-engine images
+        uses: docker/build-push-action@v2
+        with:
+          platforms: linux/${{ matrix.arch }}
+          push: false
+          load: true
+          build-args:
+            PYTHON_VERSION=${{ matrix.python_version }}
+          tags: |
+            ${{ env.DOCKER_IMAGE_TAG }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Run tests
+        if: ${{ matrix.run_tests }}
+        run: |
+          docker tag ${{ env.DOCKER_IMAGE_TAG }} ${{ env.PROJECT }}_app
+          if [[ -z $SKIP_TESTS ]]; then
+            docker-compose run app bash -ec '
+              bin/wait-for-it.sh mysql:3306 \
+              && NYLAS_ENV=test pytest --cov-report= --cov=inbox tests/ \
+              && coverage html -d pythoncov
+            '
+          else
+            echo Skipping tests
+          fi
+
+      - name: Push image
+        if: ${{ matrix.dockerhub_push }}
+        run: |
+          docker push ${{ env.DOCKER_IMAGE_TAG }}
+
+  multi-arch-manifest:
+    needs:
+      - build-arm64
+
+    runs-on: ubuntu-20.04
+
+    strategy:
+      matrix:
+        include:
+          - build_name: py2.7
+          - build_name: py3.8
+
+    steps:
+      - name: Get short sha
+        id: vars
+        run: |
+          echo ::set-output name=sha_short::${GITHUB_SHA::16}
+        shell: bash
+
+      - name: Prepare environment
+        run: |
+          if [ "$(git log -1 --pretty=%B | head -n 1 | grep '#notests')" ]; then
+            echo 'SKIP_TESTS="1"' >> $GITHUB_ENV
+          fi
+          echo "DOCKER_IMAGE_TAG=${{ env.DOCKERHUB_NAMESPACE }}/${{ env.PROJECT }}:${{ steps.vars.outputs.sha_short }}-${{ matrix.build_name}}${{ env.TAG_SUFFIX }}" >> $GITHUB_ENV
+        shell: bash
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      # Because the arm64 builds are so much slower, we assume they will just 
+      # be appended to the existing manifest created by `build-and-release.yaml`.
+      - name: Create manifest
+        run: |
+          docker buildx imagetools create --append \
+            -t ${{ env.DOCKER_IMAGE_TAG }} \
+            ${{ env.DOCKER_IMAGE_TAG }}-arm64