fix: Fix CI failure when building manylinux image (#4122)

## What do these changes do? - Return the error messages Immediately when executing the image building task through `gsctl` - Support treat `graphlearn` as an option to build `graphscope-client` wheel package Co-authored-by: xiaolei.zl <[email protected]>
alibaba · Aug 14, 2024 · 83995e0 · 83995e0
1 parent e69a05a
commit 83995e0
Show file tree

Hide file tree

Showing 26 changed files with 440 additions and 628 deletions.
diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml
diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml
@@ -0,0 +1,56 @@
+name: Build GraphScope Wheel Images
+
+# build `graphscope/graphscope-dev:wheel-{v6d_version}` image based on manylinux,
+# including all dependencies for building graphscope wheel package.
+on:
+  workflow_dispatch:
+    inputs:
+      v6d_version:
+        description: 'Version for Vineyard (v6d)'
+        required: true
+        default: 'main'
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'python/graphscope/gsctl/scripts/**'
+      - '.github/workflows/build-graphscope-dev-wheel-images.yml'
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: registry.cn-hongkong.aliyuncs.com
+
+jobs:
+  build-wheel-image-amd64:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Build Image
+      run: |
+        # build wheel image with specified v6d's version
+        cd ${GITHUB_WORKSPACE}/k8s
+        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
+        if [[ -n ${VINEYARD_VERSION} ]]; then
+          # graphscope/graphscope-dev:wheel-<v6d_version>-amd64
+          make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
+        else
+          # pull_request: use default vineyard_version
+          make dev-wheel
+        fi
+
+    - name: Release Image
+      if: ${{ github.event_name == 'workflow_dispatch' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
+        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
+        sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml
diff --git a/.github/workflows/build-graphscope-manylinux-images.yml b/.github/workflows/build-graphscope-manylinux-images.yml
@@ -0,0 +1,45 @@
+name: Build GraphScope Manylinux Images
+
+# build `graphscope/manylinux2014` image based on centos7, including all necessary
+# dependencies except vineyard
+on:
+  workflow_dispatch:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'python/graphscope/gsctl/scripts/**'
+      - '.github/workflows/build-graphscope-manylinux-images.yml'
+
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+env:
+  REGISTRY: registry.cn-hongkong.aliyuncs.com
+
+jobs:
+  build-manylinux-image-amd64:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+
+    - name: Build Image
+      run: |
+        cd ${GITHUB_WORKSPACE}/k8s
+        # output: graphscope/manylinux:amd64
+        make manylinux2014
+
+    - name: Release Image
+      if: ${{ github.event_name == 'workflow_dispatch' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
+        # manylinux2014 image
+        sudo docker tag graphscope/manylinux2014:amd64 ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64
diff --git a/.github/workflows/flex-interactive.yml b/.github/workflows/flex-interactive.yml
@@ -33,11 +33,6 @@ jobs:
     - name: Checkout code
       uses: actions/checkout@v4
 
-    - name: Add envs to GITHUB_ENV
-      run: |
-        short_sha=$(git rev-parse --short HEAD)
-        echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV
-
     - name: Build Image
       run: |
         cd ${GITHUB_WORKSPACE}
@@ -59,7 +54,7 @@ jobs:
         # install gsctl
         python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl
         # launch service: 8080 for coordinator http port; 7687 for cypher port;
-        docker run -p 8080:8080 -p 7688:7687 registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-x86_64 --enable-coordinator &
+        docker run -p 8080:8080 -p 7688:7687 graphscope/interactive:latest --enable-coordinator &
         sleep 20
         # test
         python3 -m pip install --no-cache-dir pytest pytest-xdist

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -64,23 +64,17 @@ jobs:
         password: ${{ secrets.PYPI_PASSWORD }}
         packages_dir: upload_pypi/
 
-  build-interactive-image-x86-64:
+  build-interactive-image-amd64:
     if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
 
     steps:
     - uses: actions/checkout@v4
 
-    - name: Add envs to GITHUB_ENV
-      run: |
-        short_sha=$(git rev-parse --short HEAD)
-        echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV
-
     - name: Build Interactive Image
       run: |
-        cd ${GITHUB_WORKSPACE}/python
-        python3 -m pip install --upgrade pip && python3 -m pip install -r requirements.txt && python3 setup.py build_proto
         cd ${GITHUB_WORKSPACE}
+        python3 -m pip install --upgrade pip && python3 -m pip install click
         python3 ./gsctl.py flexbuild interactive --app docker
 
     - name: Extract Tag Name
@@ -95,10 +89,10 @@ jobs:
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        sudo docker tag ${{ env.INTERACTIVE_IMAGE }}:${SHORT_SHA}-x86_64 ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}
-        sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/interactive:latest ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64
+        sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64
 
-  build-gss-image-x86-64:
+  build-gss-image-amd64:
     if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     runs-on: ubuntu-20.04
     steps:
@@ -114,25 +108,6 @@ jobs:
         cd ${GITHUB_WORKSPACE}/k8s
         sudo make graphscope-store VERSION=${SHORT_SHA}
 
-    - name: Release Nightly Image
-      # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
-      # GraphScope-Store doesn't need nightly release yet.
-      # To save some resources, comment this step out temporarily.
-      if: false
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        # docker tag: 0.15.0 -> 0.15.0a20220808
-        # x86_64
-        # arch=$(uname -m)
-        time=$(date "+%Y%m%d")
-        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        tag="${version}a${time}"
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag}
-        sudo docker push ${{ env.GSS_IMAGE }}:${tag}
-
     - name: Extract Tag Name
       if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
@@ -145,12 +120,10 @@ jobs:
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        # x86_64
-        # arch=$(uname -m)
         sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
 
-  build-gss-image-aarch64:
+  build-gss-image-arm64:
     # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     if: false
     runs-on: [self-hosted, Linux, ARM64]
@@ -167,25 +140,6 @@ jobs:
         cd ${GITHUB_WORKSPACE}/k8s
         sudo make graphscope-store VERSION=${SHORT_SHA}
 
-    - name: Release Nightly Image
-      # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
-      # GraphScope-Store doesn't need nightly release yet.
-      # To save some resources, comment this step out temporarily.
-      if: false
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        # docker tag: 0.15.0 -> 0.15.0a20220808
-        # aarch64
-        arch=$(uname -m)
-        time=$(date "+%Y%m%d")
-        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        tag="${version}a${time}"-${arch}
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag}
-        sudo docker push ${{ env.GSS_IMAGE }}:${tag}
-
     - name: Extract Tag Name
       if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
@@ -198,38 +152,15 @@ jobs:
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        # aarch64
-        arch=$(uname -m)
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch}
-        sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch}
-
-    - name: Clean Nightly Image
-      if: false
-      run: |
-        # docker tag: 0.15.0 -> 0.15.0a20220808
-        # aarch64
-        arch=$(uname -m)
-        time=$(date "+%Y%m%d")
-        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        tag="${version}a${time}"-${arch}
-        # clean
-        sudo docker rmi -f graphscope-store:${SHORT_SHA} || true
-        sudo docker rmi -f ${{ env.GSS_IMAGE }}:${tag} || true
-
-    - name: Clean Release Image
-      if: always()
-      run: |
-        # aarch64
-        arch=$(uname -m)
-        sudo docker rmi -f graphscope-store:${SHORT_SHA} || true
-        sudo docker rmi -f ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} || true
+        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64
+        sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64
 
   push-gss-image-manifest:
     # if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
     if: false
     runs-on: ubuntu-20.04
 
-    needs: [build-gss-image-x86-64, build-gss-image-aarch64]
+    needs: [build-gss-image-amd64, build-gss-image-arm64]
 
     steps:
     - uses: actions/checkout@v4
@@ -251,8 +182,8 @@ jobs:
         # create
         sudo docker manifest create \
           ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} \
-          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-x86_64 \
-          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-aarch64
+          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 \
+          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64
 
         # push
         sudo docker manifest push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}