From 83995e01df026b0bf373b832e6a3da009c4960d7 Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Wed, 14 Aug 2024 20:03:38 +0800 Subject: [PATCH] fix: Fix CI failure when building manylinux image (#4122) ## What do these changes do? - Return the error messages Immediately when executing the image building task through `gsctl` - Support treat `graphlearn` as an option to build `graphscope-client` wheel package Co-authored-by: xiaolei.zl --- .../workflows/build-graphscope-dev-images.yml | 259 ++++-------------- .../build-graphscope-dev-wheel-images.yml | 56 ++++ .../build-graphscope-manylinux-ext-images.yml | 106 ------- .../build-graphscope-manylinux-images.yml | 45 +++ .github/workflows/flex-interactive.yml | 7 +- .github/workflows/release.yml | 93 +------ flex/resources/hqps/CMakeLists.txt.template | 4 + k8s/Makefile | 45 +-- k8s/dockerfiles/analytical.Dockerfile | 9 +- k8s/dockerfiles/coordinator.Dockerfile | 3 +- k8s/dockerfiles/flex-interactive.Dockerfile | 64 +++-- k8s/dockerfiles/graphlearn-torch.Dockerfile | 5 +- .../graphscope-dev-wheel.Dockerfile | 77 ++++++ k8s/dockerfiles/graphscope-dev.Dockerfile | 8 +- k8s/dockerfiles/graphscope-store.Dockerfile | 3 +- .../interactive-experimental.Dockerfile | 3 +- k8s/dockerfiles/interactive.Dockerfile | 7 +- k8s/dockerfiles/learning.Dockerfile | 3 +- k8s/dockerfiles/manylinux2014-ext.Dockerfile | 41 --- k8s/dockerfiles/manylinux2014.Dockerfile | 106 +++---- k8s/dockerfiles/vineyard-dev.Dockerfile | 13 +- k8s/dockerfiles/vineyard-runtime.Dockerfile | 3 +- k8s/internal/Makefile | 53 ++-- python/graphscope/gsctl/commands/dev.py | 3 + .../gsctl/scripts/install_deps_command.sh | 22 +- python/setup.py | 30 +- 26 files changed, 440 insertions(+), 628 deletions(-) create mode 100644 .github/workflows/build-graphscope-dev-wheel-images.yml delete mode 100644 .github/workflows/build-graphscope-manylinux-ext-images.yml create mode 100644 .github/workflows/build-graphscope-manylinux-images.yml create mode 100644 k8s/dockerfiles/graphscope-dev-wheel.Dockerfile delete mode 100644 k8s/dockerfiles/manylinux2014-ext.Dockerfile diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml index c47810203b8a..22eba205b978 100644 --- a/.github/workflows/build-graphscope-dev-images.yml +++ b/.github/workflows/build-graphscope-dev-images.yml @@ -1,10 +1,9 @@ -name: Build GraphScope Development and Wheel Images +name: Build GraphScope Development Images # build images for: -# 1) wheel: including all dependencies for graphscope's wheel package. -# 2) graphscope-dev: including all dependencies for graphscope's development env. -# 3) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. -# 4) vineyard-runtime: including all vineyard-related running dependencies. +# 1) graphscope-dev: including all dependencies for graphscope development env. +# 2) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. +# 3) vineyard-runtime: including all vineyard-related running dependencies. # Note that: # Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github. on: @@ -14,11 +13,6 @@ on: description: 'Version for Vineyard (v6d)' required: true default: 'main' - build_wheel: - description: 'Whether to build dev-wheel image' - required: true - default: true - type: boolean build_graphscope_dev: description: 'Whether to build graphscope-dev image' required: true @@ -34,6 +28,7 @@ on: - main paths: - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-dev-images.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -43,116 +38,9 @@ env: REGISTRY: registry.cn-hongkong.aliyuncs.com jobs: - build-wheel-image-x86-64: - runs-on: ubuntu-20.04 - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request') - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--x86_64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - build-wheel-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request') - if: false - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--aarch64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:wheel-${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} || true - - manifest-push-wheel-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_wheel == 'true' }} - if: false - - needs: [build-wheel-image-x86-64, build-wheel-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} - - build-graphscope-dev-image-x86-64: - runs-on: ubuntu-20.04 + build-graphscope-dev-image-amd64: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') - + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -160,11 +48,11 @@ jobs: - name: Build Image run: | - # build graphscope dev image with specified v6d's version + # build graphscope dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-x86_64 + # graphscope/graphscope-dev:-amd64 make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -178,19 +66,13 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 - build-graphscope-dev-image-aarch64: + build-graphscope-dev-image-arm64: runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') if: false - steps: - uses: actions/checkout@v4 with: @@ -198,11 +80,11 @@ jobs: - name: Build Image run: | - # build graphscope dev image with specified v6d's version + # build graphscope dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-aarch64 + # graphscope/graphscope-dev:-arm64 make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -216,30 +98,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} || true + sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 manifest-push-graphscope-dev-image: runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_graphscope_dev == 'true' }} if: false - - needs: [build-graphscope-dev-image-x86-64, build-graphscope-dev-image-aarch64] + needs: [build-graphscope-dev-image-amd64, build-graphscope-dev-image-arm64] steps: - name: Create and Push Docker Manifest env: @@ -250,15 +117,14 @@ jobs: # manifest create sudo docker manifest create \ ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-aarch64 + ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-amd64 \ + ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-arm64 # manifest push sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} - build-vineyard-dev-image-x86-64: - runs-on: ubuntu-20.04 + build-vineyard-dev-image-amd64: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') - + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -266,11 +132,11 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-dev:-x86_64 + # graphscope/vineyard-dev:-amd64 make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -284,19 +150,13 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 - build-vineyard-dev-image-aarch64: + build-vineyard-dev-image-arm64: runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') if: false - steps: - uses: actions/checkout@v4 with: @@ -304,7 +164,7 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then @@ -322,30 +182,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/vineyard-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} || true + sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 manifest-push-vineyard-dev-image: runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} if: false - - needs: [build-vineyard-dev-image-x86-64, build-vineyard-dev-image-aarch64] + needs: [build-vineyard-dev-image-amd64, build-vineyard-dev-image-arm64] steps: - name: Create and Push Docker Manifest env: @@ -356,18 +201,16 @@ jobs: # manifest create sudo docker manifest create \ ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-aarch64 + ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-amd64 \ + ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-arm64 # manifest push sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} - build-vineyard-runtime-image-x86-64: - runs-on: ubuntu-20.04 - # only trigger this step in 'workflow_dispatch' event, - # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' + build-vineyard-runtime-image-amd64: + # only trigger this step in 'workflow_dispatch' event, since the 'vineyard-dev' image isn't actually pushed in 'pull_request' if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request') - - needs: [build-vineyard-dev-image-x86-64] + runs-on: ubuntu-20.04 + needs: [build-vineyard-dev-image-amd64] steps: - uses: actions/checkout@v4 with: @@ -375,11 +218,11 @@ jobs: - name: Build Image run: | - # build vineyard runtime image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + # build vineyard runtime image with specified v6d version + cd ${GITHUB_WORKSPACE}/k8s if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-runtime:-x86_64 + # graphscope/vineyard-runtime:-amd64 make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -393,15 +236,11 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-runtime image - sudo docker tag graphscope/vineyard-runtime:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 - build-vineyard-runtime-image-aarch64: + build-vineyard-runtime-image-arm64: runs-on: [self-hosted, Linux, ARM64] # only trigger this step in 'workflow_dispatch' event, # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' @@ -416,7 +255,7 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then @@ -456,7 +295,7 @@ jobs: # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} if: false - needs: [build-vineyard-runtime-image-x86-64, build-vineyard-runtime-image-aarch64] + needs: [build-vineyard-runtime-image-amd64, build-vineyard-runtime-image-arm64] steps: - name: Create and Push Docker Manifest env: diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml new file mode 100644 index 000000000000..791a88e4a1fa --- /dev/null +++ b/.github/workflows/build-graphscope-dev-wheel-images.yml @@ -0,0 +1,56 @@ +name: Build GraphScope Wheel Images + +# build `graphscope/graphscope-dev:wheel-{v6d_version}` image based on manylinux, +# including all dependencies for building graphscope wheel package. +on: + workflow_dispatch: + inputs: + v6d_version: + description: 'Version for Vineyard (v6d)' + required: true + default: 'main' + pull_request: + branches: + - main + paths: + - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-dev-wheel-images.yml' + +concurrency: + group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +env: + REGISTRY: registry.cn-hongkong.aliyuncs.com + +jobs: + build-wheel-image-amd64: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build Image + run: | + # build wheel image with specified v6d's version + cd ${GITHUB_WORKSPACE}/k8s + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + if [[ -n ${VINEYARD_VERSION} ]]; then + # graphscope/graphscope-dev:wheel--amd64 + make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} + else + # pull_request: use default vineyard_version + make dev-wheel + fi + + - name: Release Image + if: ${{ github.event_name == 'workflow_dispatch' }} + env: + docker_password: ${{ secrets.DOCKER_PASSWORD }} + docker_username: ${{ secrets.DOCKER_USER }} + run: | + echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml deleted file mode 100644 index 0c2844ab23ae..000000000000 --- a/.github/workflows/build-graphscope-manylinux-ext-images.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Build GraphScope Manylinux-Ext Images - -# build graphscope-dev-base image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. -on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - 'python/graphscope/gsctl/scripts/**' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-manylinux-ext-image-x86-64: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux:ext-x86_64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - - build-manylinux-ext-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - if: false - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux2014:ext-aarch64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # clean - sudo docker rmi -f graphscope/manylinux2014:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} || true - - manifest_push_manylinux-ext-image: - runs-on: ubuntu-20.04 - if: false - needs: [build-manylinux-ext-image-x86-64, build-manylinux-ext-image-aarch64] - steps: - - name: Create and Push Docker Manifest - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-x86_64 \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-aarch64 - # push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext diff --git a/.github/workflows/build-graphscope-manylinux-images.yml b/.github/workflows/build-graphscope-manylinux-images.yml new file mode 100644 index 000000000000..123dc1213335 --- /dev/null +++ b/.github/workflows/build-graphscope-manylinux-images.yml @@ -0,0 +1,45 @@ +name: Build GraphScope Manylinux Images + +# build `graphscope/manylinux2014` image based on centos7, including all necessary +# dependencies except vineyard +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-manylinux-images.yml' + + +concurrency: + group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +env: + REGISTRY: registry.cn-hongkong.aliyuncs.com + +jobs: + build-manylinux-image-amd64: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build Image + run: | + cd ${GITHUB_WORKSPACE}/k8s + # output: graphscope/manylinux:amd64 + make manylinux2014 + + - name: Release Image + if: ${{ github.event_name == 'workflow_dispatch' }} + env: + docker_password: ${{ secrets.DOCKER_PASSWORD }} + docker_username: ${{ secrets.DOCKER_USER }} + run: | + echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin + # manylinux2014 image + sudo docker tag graphscope/manylinux2014:amd64 ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64 diff --git a/.github/workflows/flex-interactive.yml b/.github/workflows/flex-interactive.yml index 18555bb44fb7..37ca3a52fd53 100644 --- a/.github/workflows/flex-interactive.yml +++ b/.github/workflows/flex-interactive.yml @@ -33,11 +33,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - name: Build Image run: | cd ${GITHUB_WORKSPACE} @@ -59,7 +54,7 @@ jobs: # install gsctl python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl # launch service: 8080 for coordinator http port; 7687 for cypher port; - docker run -p 8080:8080 -p 7688:7687 registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-x86_64 --enable-coordinator & + docker run -p 8080:8080 -p 7688:7687 graphscope/interactive:latest --enable-coordinator & sleep 20 # test python3 -m pip install --no-cache-dir pytest pytest-xdist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77adb6251a79..df8b4d74887e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,23 +64,17 @@ jobs: password: ${{ secrets.PYPI_PASSWORD }} packages_dir: upload_pypi/ - build-interactive-image-x86-64: + build-interactive-image-amd64: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - name: Build Interactive Image run: | - cd ${GITHUB_WORKSPACE}/python - python3 -m pip install --upgrade pip && python3 -m pip install -r requirements.txt && python3 setup.py build_proto cd ${GITHUB_WORKSPACE} + python3 -m pip install --upgrade pip && python3 -m pip install click python3 ./gsctl.py flexbuild interactive --app docker - name: Extract Tag Name @@ -95,10 +89,10 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - sudo docker tag ${{ env.INTERACTIVE_IMAGE }}:${SHORT_SHA}-x86_64 ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }} - sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }} + sudo docker tag graphscope/interactive:latest ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 + sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 - build-gss-image-x86-64: + build-gss-image-amd64: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') runs-on: ubuntu-20.04 steps: @@ -114,25 +108,6 @@ jobs: cd ${GITHUB_WORKSPACE}/k8s sudo make graphscope-store VERSION=${SHORT_SHA} - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # x86_64 - # arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}" - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - name: Extract Tag Name if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} id: tag @@ -145,12 +120,10 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # x86_64 - # arch=$(uname -m) sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} - build-gss-image-aarch64: + build-gss-image-arm64: # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') if: false runs-on: [self-hosted, Linux, ARM64] @@ -167,25 +140,6 @@ jobs: cd ${GITHUB_WORKSPACE}/k8s sudo make graphscope-store VERSION=${SHORT_SHA} - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - name: Extract Tag Name if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} id: tag @@ -198,38 +152,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # aarch64 - arch=$(uname -m) - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - - - name: Clean Nightly Image - if: false - run: | - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - # clean - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${tag} || true - - - name: Clean Release Image - if: always() - run: | - # aarch64 - arch=$(uname -m) - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} || true + sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 + sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 push-gss-image-manifest: # if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} if: false runs-on: ubuntu-20.04 - needs: [build-gss-image-x86-64, build-gss-image-aarch64] + needs: [build-gss-image-amd64, build-gss-image-arm64] steps: - uses: actions/checkout@v4 @@ -251,8 +182,8 @@ jobs: # create sudo docker manifest create \ ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-aarch64 + ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 \ + ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 # push sudo docker manifest push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} diff --git a/flex/resources/hqps/CMakeLists.txt.template b/flex/resources/hqps/CMakeLists.txt.template index 67fba9b92490..563736b77a6d 100644 --- a/flex/resources/hqps/CMakeLists.txt.template +++ b/flex/resources/hqps/CMakeLists.txt.template @@ -38,6 +38,10 @@ if(EXISTS "/opt/graphscope/include") include_directories("/opt/graphscope/include") endif() +if (EXISTS "/opt/flex/include") + include_directories("/opt/flex/include") +endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O2 -flto -Werror=unused-result -fPIC -no-pie") diff --git a/k8s/Makefile b/k8s/Makefile index 8657b4b52f47..4dfb2e9fdb47 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -6,8 +6,10 @@ ifeq ($(REGISTRY),) REGISTRY := registry.cn-hongkong.aliyuncs.com endif -# x86_64 or aarch64 -ARCH := $(shell uname -m) +PLATFORM := $(shell uname -m) +# can be: x86_64, arm64 +ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) + VERSION ?= latest VINEYARD_VERSION ?= v0.23.0 @@ -41,36 +43,34 @@ interactive: interactive-frontend interactive-executor all: coordinator analytical interactive learning graphscope: all +manylinux2014: + cd $(WORKING_DIR)/../ && \ + docker build \ + -t graphscope/manylinux2014:${ARCH} \ + -f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile . + dev-wheel: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ - -f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile . - - -manylinux2014-ext: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ - docker build \ - -t graphscope/manylinux2014:ext-${ARCH} \ - -f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile . + -f $(DOCKERFILES_DIR)/graphscope-dev-wheel.Dockerfile . graphscope-dev: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/graphscope-dev:${VINEYARD_VERSION}-${ARCH} \ -f $(DOCKERFILES_DIR)/graphscope-dev.Dockerfile . vineyard-dev: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/vineyard-dev:${VINEYARD_VERSION}-${ARCH} \ @@ -79,6 +79,7 @@ vineyard-dev: vineyard-runtime: cd $(WORKING_DIR) && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ -t graphscope/vineyard-runtime:${VINEYARD_VERSION}-${ARCH} \ @@ -88,6 +89,7 @@ coordinator: cd $(WORKING_DIR)/.. && \ docker build \ --target coordinator \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg CI=${CI} \ @@ -98,6 +100,7 @@ analytical: cd $(WORKING_DIR)/.. && \ docker build \ --target analytical \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -109,6 +112,7 @@ analytical-java: cd $(WORKING_DIR)/.. && \ docker build \ --target analytical-java \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -120,6 +124,7 @@ interactive-frontend: cd $(WORKING_DIR)/.. && \ docker build \ --target frontend \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -132,6 +137,7 @@ interactive-executor: cd $(WORKING_DIR)/.. \ && docker build \ --target executor \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -145,6 +151,7 @@ interactive-experimental: cd $(WORKING_DIR)/.. && \ docker build \ --target experimental \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ -t graphscope/interactive-experimental:${VERSION} \ @@ -158,6 +165,7 @@ flex-interactive: --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg PLATFORM=${PLATFORM} \ --build-arg ARCH=${ARCH} \ -t graphscope/interactive:${VERSION} \ -f ${DOCKERFILES_DIR}/flex-interactive.Dockerfile . @@ -166,6 +174,7 @@ learning: cd $(WORKING_DIR)/.. && \ docker build \ --target learning \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -177,6 +186,7 @@ graphlearn-torch: cd $(WORKING_DIR)/.. && \ docker build \ --target graphlearn-torch \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -187,6 +197,7 @@ graphlearn-torch: graphscope-store: cd $(WORKING_DIR)/.. && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg profile=${PROFILE} \ diff --git a/k8s/dockerfiles/analytical.Dockerfile b/k8s/dockerfiles/analytical.Dockerfile index 9cc0614861aa..c2cfc8a248b9 100644 --- a/k8s/dockerfiles/analytical.Dockerfile +++ b/k8s/dockerfiles/analytical.Dockerfile @@ -1,10 +1,11 @@ # Analytical engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest ############### BUILDER: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false @@ -26,7 +27,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION AS analytical +FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical ENV GRAPHSCOPE_HOME=/opt/graphscope ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GRAPHSCOPE_HOME/lib @@ -58,7 +59,7 @@ COPY ./k8s/dockerfiles/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ############### BUILDER: ANALYTICAL-JAVA ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder-java +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder-java COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -81,7 +82,7 @@ RUN cd /home/graphscope/GraphScope/ && \ FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION AS analytical-java +FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical-java COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 ENV LLVM11_HOME=/opt/llvm11 ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config diff --git a/k8s/dockerfiles/coordinator.Dockerfile b/k8s/dockerfiles/coordinator.Dockerfile index a2719ce6e68f..2eff6059a2cf 100644 --- a/k8s/dockerfiles/coordinator.Dockerfile +++ b/k8s/dockerfiles/coordinator.Dockerfile @@ -1,8 +1,9 @@ # Coordinator of graphscope engines +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index 693179a5f3d2..ed852bdc4f5b 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -1,9 +1,10 @@ # Coordinator of graphscope engines +ARG PLATFORM=x86_64 +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder -ARG ARCH=x86_64 +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG ENABLE_COORDINATOR="false" RUN sudo mkdir -p /opt/flex && sudo chown -R graphscope:graphscope /opt/flex/ @@ -53,6 +54,9 @@ RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ python3 -m pip install --upgrade pip && python3 -m pip install -r requirements.txt && \ python3 setup.py build_proto && python3 setup.py bdist_wheel && \ mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/ && \ + cd ${HOME}/GraphScope/python && \ + export WITHOUT_LEARNING_ENGINE=ON && python3 setup.py bdist_wheel && \ + mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/ && \ cd ${HOME}/GraphScope/coordinator && \ python3 setup.py bdist_wheel && \ mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/; \ @@ -62,7 +66,7 @@ RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ ########################### RUNTIME IMAGE ########################### from ubuntu:22.04 as runtime -ARG ARCH +ARG PLATFORM=x86_64 ARG ENABLE_COORDINATOR="false" ENV DEBIAN_FRONTEND=noninteractive @@ -89,9 +93,13 @@ ENV LC_ALL en_US.UTF-8 ENV TZ=Asia/Shanghai RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN mkdir /opt/vineyard/ + # copy builder's /opt/flex to final image COPY --from=builder /opt/flex /opt/flex COPY --from=builder /opt/graphscope/lib/libgrape-lite.so /opt/flex/lib/ +COPY --from=builder /opt/graphscope/include/ /opt/flex/include/ +COPY --from=builder /opt/vineyard/include/ /opt/vineyard/include/ # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ @@ -106,42 +114,42 @@ RUN sed -i 's/default_graph: modern_graph/default_graph: gs_interactive_default_ # remove bin/run_app RUN rm -rf /opt/flex/bin/run_app -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libsnappy*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libsnappy*.so* /usr/lib/$PLATFORM-linux-gnu/ COPY --from=builder /usr/include/arrow /usr/include/arrow COPY --from=builder /usr/include/yaml-cpp /usr/include/yaml-cpp -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libgflags*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libglog*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libyaml-cpp*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libmpi*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_program_options*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_filesystem*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_thread*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libcrypto*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libopen-rte*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libhwloc*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libgflags*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libglog*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libyaml-cpp*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libmpi*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_program_options*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_filesystem*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_thread*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libcrypto*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libopen-rte*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libhwloc*.so* /usr/lib/$PLATFORM-linux-gnu/ # libunwind for arm64 seems not installed here, and seems not needed for aarch64(tested) -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libunwind*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libarrow.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libopen-pal*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libltdl*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libevent*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libutf8proc*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libre2*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libunwind*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libarrow.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libopen-pal*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libltdl*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libevent*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libutf8proc*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libre2*.so* /usr/lib/$PLATFORM-linux-gnu/ COPY --from=builder /usr/include/glog /usr/include/glog COPY --from=builder /usr/include/gflags /usr/include/gflags -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libprotobuf* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libfmt*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libprotobuf* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libfmt*.so* /usr/lib/$PLATFORM-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/openmpi/include/ /opt/flex/include +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/openmpi/include/ /opt/flex/include COPY --from=builder /usr/include/boost /usr/include/boost COPY --from=builder /usr/include/google /usr/include/google COPY --from=builder /usr/include/yaml-cpp /usr/include/yaml-cpp -RUN sudo rm -rf /usr/lib/$ARCH-linux-gnu/libLLVM*.so* && sudo rm -rf /opt/flex/lib/libseastar.a && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libcuda.so && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libcudart.so && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libicudata.so* +RUN sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/libLLVM*.so* && sudo rm -rf /opt/flex/lib/libseastar.a && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libcuda.so && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libcudart.so && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libicudata.so* RUN sudo ln -sf /opt/flex/bin/* /usr/local/bin/ \ && sudo ln -sfn /opt/flex/include/* /usr/local/include/ \ diff --git a/k8s/dockerfiles/graphlearn-torch.Dockerfile b/k8s/dockerfiles/graphlearn-torch.Dockerfile index 8a763a3499ba..970eebc2282e 100644 --- a/k8s/dockerfiles/graphlearn-torch.Dockerfile +++ b/k8s/dockerfiles/graphlearn-torch.Dockerfile @@ -1,9 +1,10 @@ # Graphlearn-torch engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=v0.21.3 ARG RUNTIME_VERSION=v0.21.3 -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -24,7 +25,7 @@ RUN cd /home/graphscope/GraphScope/; \ cp dist/*.whl /home/graphscope/install/ ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS graphlearn-torch +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS graphlearn-torch RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile new file mode 100644 index 000000000000..1a58a42839f8 --- /dev/null +++ b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile @@ -0,0 +1,77 @@ +# build `graphscope/graphscope-dev:wheel-{v6d_version}-{arch}` image based on manylinux, +# including all dependencies for building graphscope wheel package. + +ARG ARCH=amd64 +ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +FROM $REGISTRY/graphscope/manylinux2014:$ARCH AS builder + +# build form https://github.com/sighingnow/manylinux/tree/dyn-rebase +# usually we don't need to change this image unless the underlying python needs to be updated +FROM $REGISTRY/graphscope/manylinux2014:20230407 + +# change the source +RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^#.*baseurl=http/baseurl=http/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^mirrorlist=http/#mirrorlist=http/g" /etc/yum.repos.d/*.repo + +# shanghai zoneinfo +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo '$TZ' > /etc/timezone + +# for programming output +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 + +ENV GRAPHSCOPE_HOME=/opt/graphscope + +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$GRAPHSCOPE_HOME/lib:$GRAPHSCOPE_HOME/lib64 +ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:/home/graphscope/.local/bin:/home/graphscope/.cargo/bin + +ENV JAVA_HOME=/usr/lib/jvm/java +ENV RUST_BACKTRACE=1 + +# install clang-11 with gold optimizer plugin, depends on header include/plugin-api.h +# COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 +# ENV LLVM11_HOME=/opt/llvm11 +# ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config + +# Copy the thirdparty c++ dependencies, maven, and hadoop +COPY --from=builder /opt/graphscope /opt/graphscope +COPY --from=builder /opt/openmpi /opt/openmpi +RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* + +RUN useradd -m graphscope -u 1001 \ + && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# Install jdk-11 +RUN yum install -y sudo vim && \ + yum install python3-pip -y && \ + yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ + yum install java-11-openjdk-devel -y && \ + yum clean all -y --enablerepo='*' && \ + rm -rf /var/cache/yum + +RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /opt/graphscope /opt/vineyard + +USER graphscope +WORKDIR /home/graphscope + +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope +ARG VINEYARD_VERSION=main +RUN cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging --user && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + sudo rm -rf /home/graphscope/GraphScope && \ + sudo yum clean all -y && \ + sudo rm -fr /var/cache/yum +RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc + +SHELL [ "/usr/bin/scl", "enable", "rh-python38" ] + +RUN python3 -m pip --no-cache install pyyaml --user +# Uncomment this line will results in a weird error when using the image together with commands, like +# docker run --rm graphscope/graphscope-dev:latest bash -c 'echo xxx && ls -la' +# The output of `ls -la` would not be shown. +# ENTRYPOINT ["/bin/bash", "-c", ". scl_source enable devtoolset-8 rh-python38 && $0 $@"] diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index b3df8bef7c6d..1c13a487033d 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -27,13 +27,13 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ +RUN cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ cd /home/graphscope && \ - rm -fr gsctl + rm -fr GraphScope RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc RUN python3 -m pip --no-cache install pyyaml ipython --user diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index d8156e50e3b9..19b694b3f085 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -1,6 +1,7 @@ +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION as builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH as builder ARG CI=false ARG ENABLE_COORDINATOR=false diff --git a/k8s/dockerfiles/interactive-experimental.Dockerfile b/k8s/dockerfiles/interactive-experimental.Dockerfile index a1cf940bd7eb..478fa3bf60e5 100644 --- a/k8s/dockerfiles/interactive-experimental.Dockerfile +++ b/k8s/dockerfiles/interactive-experimental.Dockerfile @@ -1,8 +1,9 @@ # Interactive engine which uses experimental storage +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index bbb8c305679d..5ab58e8bd340 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -1,9 +1,10 @@ # Interactive engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false @@ -52,8 +53,8 @@ USER graphscope WORKDIR /home/graphscope ############### RUNTIME: executor ####################### -FROM registry.cn-hongkong.aliyuncs.com/graphscope/manylinux2014:ext AS ext -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS executor +FROM registry.cn-hongkong.aliyuncs.com/graphscope/manylinux2014:$ARCH AS ext +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS executor ENV RUST_BACKTRACE=1 diff --git a/k8s/dockerfiles/learning.Dockerfile b/k8s/dockerfiles/learning.Dockerfile index 7edbffa86e80..20b55f08fd67 100644 --- a/k8s/dockerfiles/learning.Dockerfile +++ b/k8s/dockerfiles/learning.Dockerfile @@ -1,5 +1,6 @@ # Learning engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest @@ -29,7 +30,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS learning +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS learning RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/manylinux2014-ext.Dockerfile b/k8s/dockerfiles/manylinux2014-ext.Dockerfile deleted file mode 100644 index 80fbbeeed3c6..000000000000 --- a/k8s/dockerfiles/manylinux2014-ext.Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# the graphscope-dev-base image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. - -FROM centos:7 AS builder - - -# shanghai zoneinfo -ENV TZ=Asia/Shanghai -RUN yum install sudo -y && \ - yum update glibc-common -y && \ - sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ - yum install python3-pip -y && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo '$TZ' > /etc/timezone - -ENV LC_ALL=en_US.utf-8 -ENV LANG=en_US.utf-8 - -COPY gsctl ./gsctl -RUN cd ./gsctl && \ - python3 -m pip install click && \ - python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ - rm -fr /root/gsctl - -# install hadoop for processing hadoop data source -RUN if [ "$(uname -m)" = "aarch64" ]; then \ - curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz; \ - tar xzf hadoop-3.3.0-aarch64.tar.gz -C /opt/; \ - else \ - curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz; \ - tar xzf hadoop-3.3.0.tar.gz -C /opt/; \ - fi && \ - rm -rf hadoop-3.3.0* && \ - cd /opt/hadoop-3.3.0/share/ && \ - rm -rf doc hadoop/client hadoop/mapreduce hadoop/tools hadoop/yarn - -FROM centos:7 - -COPY --from=builder /opt/graphscope /opt/graphscope -COPY --from=builder /opt/openmpi /opt/openmpi -COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index c8ba01c6930d..fe2e85349658 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -1,73 +1,45 @@ -# the manylinux2014 image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. -# It's tagged as the graphscope/graphscope-dev:wheel +# build `graphscope/manylinux2014` image based on centos7, including all necessary +# dependencies except vineyard +FROM centos:7 AS builder -ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -#FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm -FROM $REGISTRY/graphscope/manylinux2014:ext AS ext - -# build form https://github.com/sighingnow/manylinux/tree/dyn-rebase -# usually we don't need to change this image unless the underlying python needs to be updated -FROM $REGISTRY/graphscope/manylinux2014:20230407 +# change the source +RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^#.*baseurl=http/baseurl=http/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^mirrorlist=http/#mirrorlist=http/g" /etc/yum.repos.d/*.repo # shanghai zoneinfo ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo '$TZ' > /etc/timezone - -# for programming output -RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 - -ENV GRAPHSCOPE_HOME=/opt/graphscope - -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$GRAPHSCOPE_HOME/lib:$GRAPHSCOPE_HOME/lib64 -ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:/home/graphscope/.local/bin:/home/graphscope/.cargo/bin - -ENV JAVA_HOME=/usr/lib/jvm/java -ENV RUST_BACKTRACE=1 - -# install clang-11 with gold optimizer plugin, depends on header include/plugin-api.h -# COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 -# ENV LLVM11_HOME=/opt/llvm11 -# ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config - -# Copy the thirdparty c++ dependencies, maven, and hadoop -COPY --from=ext /opt/graphscope /opt/graphscope -COPY --from=ext /opt/openmpi /opt/openmpi -RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* - -RUN useradd -m graphscope -u 1001 \ - && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# Install jdk-11 -RUN yum install -y sudo vim && \ +RUN yum install sudo -y && \ + yum update glibc-common -y && \ + sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ yum install python3-pip -y && \ - yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ - yum install java-11-openjdk-devel -y && \ - yum clean all -y --enablerepo='*' && \ - rm -rf /var/cache/yum - -RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /opt/graphscope /opt/vineyard - -USER graphscope -WORKDIR /home/graphscope - -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl -ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ - sudo python3 -m pip install click && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl && \ - sudo yum clean all -y && \ - sudo rm -fr /var/cache/yum -RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc - -SHELL [ "/usr/bin/scl", "enable", "rh-python38" ] + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo '$TZ' > /etc/timezone -RUN python3 -m pip --no-cache install pyyaml --user -# Uncomment this line will results in a weird error when using the image together with commands, like -# docker run --rm graphscope/graphscope-dev:latest bash -c 'echo xxx && ls -la' -# The output of `ls -la` would not be shown. -# ENTRYPOINT ["/bin/bash", "-c", ". scl_source enable devtoolset-8 rh-python38 && $0 $@"] +ENV LC_ALL=en_US.utf-8 +ENV LANG=en_US.utf-8 + +COPY . /root/GraphScope +RUN cd /root/GraphScope && \ + python3 -m pip install click packaging && \ + python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ + rm -fr /root/GraphScope + +# install hadoop for processing hadoop data source +RUN if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ + curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz; \ + tar xzf hadoop-3.3.0-aarch64.tar.gz -C /opt/; \ + else \ + curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz; \ + tar xzf hadoop-3.3.0.tar.gz -C /opt/; \ + fi && \ + rm -rf hadoop-3.3.0* && \ + cd /opt/hadoop-3.3.0/share/ && \ + rm -rf doc hadoop/client hadoop/mapreduce hadoop/tools hadoop/yarn + + +FROM centos:7 + +COPY --from=builder /opt/graphscope /opt/graphscope +COPY --from=builder /opt/openmpi /opt/openmpi +COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index 7ef99f9a1a53..a68df0989273 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -1,7 +1,8 @@ # The vineyard-dev image including all vineyard-related dependencies # that could compile graphscope analytical engine. +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -FROM $REGISTRY/graphscope/manylinux2014:ext AS ext +FROM $REGISTRY/graphscope/manylinux2014:$ARCH AS builder FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive @@ -21,7 +22,7 @@ ENV HADOOP_YARN_HOME=$HADOOP_HOME HADOOP_MAPRED_HOME=$HADOOP_HOME ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:$HADOOP_HOME/bin:/home/graphscope/.local/bin # Copy hadoop -COPY --from=ext /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 +COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 RUN apt-get update && \ apt-get install -y sudo default-jre python3-pip tzdata openssh-server dnsutils && \ @@ -38,13 +39,13 @@ RUN mkdir -p /var/log/graphscope && chown -R graphscope:graphscope /var/log/grap USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ - cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ + cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl + cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope RUN python3 -m pip --no-cache install pyyaml --user diff --git a/k8s/dockerfiles/vineyard-runtime.Dockerfile b/k8s/dockerfiles/vineyard-runtime.Dockerfile index 2c3d0c0cb598..e619113afdc7 100644 --- a/k8s/dockerfiles/vineyard-runtime.Dockerfile +++ b/k8s/dockerfiles/vineyard-runtime.Dockerfile @@ -1,9 +1,10 @@ # The vineyard-runtime image including all vineyard-related # dependencies that could graphscope interactive engine. +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/vineyard-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/vineyard-dev:$BUILDER_VERSION-$ARCH AS builder FROM ubuntu:22.04 AS runtime diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index 0260da202275..a3025067e5f7 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -17,8 +17,9 @@ ifeq ($(UNAME_S), Linux) XARGS_EMPTY_FLAG := --no-run-if-empty endif -# x86_64 or arm64 -ARCH := $(shell uname -m) +PLATFORM := $(shell uname -m) +# can be: x86_64, arm64 +ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) MACOS_WHEEL_VERSION := 12_0 # docker build arguments @@ -55,7 +56,7 @@ graphscope: graphscope-jupyter-manylinux2014-py3: docker run --rm -it -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c 'pip3 install graphscope-client -U && \ cd /work/python/jupyter/graphscope && \ python3 setup.py bdist_wheel' @@ -81,21 +82,21 @@ graphscope-darwin-py3: # build and delocate wheel cd $(WORKING_DIR)/../../coordinator && \ export WITH_EXTRA_DATA=ON && \ - if [[ "${ARCH}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \ + if [[ "${PLATFORM}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \ rm -rf build dist/*.whl || true && \ sudo strip -s $(WORKING_DIR)/../../analytical_engine/exported_symbols_osx.lds ${INSTALL_PREFIX}/bin/grape_engine || true && \ sudo strip ${INSTALL_PREFIX}/bin/gaia_executor && \ export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \ install_name_tool -add_rpath /usr/local/lib ${INSTALL_PREFIX}/bin/gaia_executor && \ - package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ pip3 install delocate==0.10.7 && \ for wheel in `ls dist/*.whl`; do \ delocate-listdeps -a -d $$wheel; \ @@ -109,8 +110,8 @@ graphscope-manylinux2014-py3-nodocker: sudo yum install java-11-openjdk-devel -y && \ sudo yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ cd $(WORKING_DIR)/../.. && \ - if [[ "${ARCH}" == "aarch64" ]]; then \ - export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ + export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ fi && \ @@ -143,13 +144,13 @@ graphscope-manylinux2014-py3-nodocker: package_name=graphscope python3 setup.py bdist_wheel && \ cd dist && \ for wheel in `ls ./*.whl`; do \ - (auditwheel repair $$wheel --plat=manylinux2014_${ARCH} && rm $$wheel) || true; \ + (auditwheel repair $$wheel --plat=manylinux2014_${PLATFORM} && rm $$wheel) || true; \ done graphscope-manylinux2014-py3: - docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} + docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} docker run --rm -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c '. ~/.graphscope_env && \ sudo chown -R graphscope /work && \ cd /work/k8s/internal && \ @@ -170,14 +171,14 @@ graphscope-client-manylinux2014-py3-nodocker: cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \ make graphlearn_shared -j`nproc` && \ export LD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:$$LD_LIBRARY_PATH && \ - if [[ "${ARCH}" == "aarch64" ]]; then export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; fi && \ + if [[ "${PLATFORM}" == "aarch64" ]]; then export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; fi && \ for py in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do \ cd $(WORKING_DIR)/../../python; \ export PATH=/opt/python/$$py/bin:$$PATH; \ python3 -m pip install ${PIP_ARGS} -U pip; \ python3 -m pip install ${PIP_ARGS} "torch<=2.2.1" --index-url https://download.pytorch.org/whl/cpu --user; \ if [[ "$$py" == "cp311-cp311" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -185,7 +186,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.23.2" "pandas" wheel "auditwheel==5.2.0"; \ elif [[ "$$py" == "cp310-cp310" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -193,7 +194,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.21.2" "pandas" wheel "auditwheel==5.2.0"; \ elif [[ "$$py" == "cp39-cp39" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -201,7 +202,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.19.3" "pandas" wheel "auditwheel==5.2.0"; \ else \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -214,7 +215,7 @@ graphscope-client-manylinux2014-py3-nodocker: python3 setup.py build_gltorch_ext; \ python3 setup.py bdist_wheel; \ cd dist; \ - auditwheel repair ./*.whl --plat=manylinux2014_${ARCH} --exclude libtorch_cpu.so --exclude libc10.so --exclude libtorch_python.so --exclude libtorch.so; \ + auditwheel repair ./*.whl --plat=manylinux2014_${PLATFORM} --exclude libtorch_cpu.so --exclude libc10.so --exclude libtorch_python.so --exclude libtorch.so; \ done graphscope-client-darwin-py3: @@ -241,29 +242,29 @@ graphscope-client-darwin-py3: elif [[ "$$py" == "310" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ elif [[ "$$py" == "39" ]]; then \ - if [[ "${ARCH}" == "arm64" ]]; then \ + if [[ "${PLATFORM}" == "arm64" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ else \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ fi; \ else \ - if [[ "${ARCH}" == "arm64" ]]; then \ + if [[ "${PLATFORM}" == "arm64" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ else \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ fi; \ fi; \ rm -rf build dist/*.whl || true && \ - python3 setup.py build_gltorch_ext --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ - python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + python3 setup.py build_gltorch_ext --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ + python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ for wheel in `ls dist/*.whl`; do \ delocate-wheel -e libtorch_cpu.dylib -e libc10.dylib -e libtorch_python.dylib -e libtorch.dylib -w dist/wheelhouse -v $$wheel && rm $$wheel; \ done graphscope-client-manylinux2014-py3: - docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} + docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} docker run --rm -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c '. ~/.graphscope_env && \ sudo chown -R graphscope /work && \ cd /work/k8s/internal && \ diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 6a5d7968ec97..8138ebb14b62 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -21,6 +21,7 @@ import io import os import subprocess +import sys import click from packaging import version @@ -62,6 +63,8 @@ def run_shell_cmd(cmd, workingdir): ) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): print(line.rstrip()) + proc.wait() + sys.exit(proc.returncode) @click.group() diff --git a/python/graphscope/gsctl/scripts/install_deps_command.sh b/python/graphscope/gsctl/scripts/install_deps_command.sh index 0925aceb458f..a835b690f8cd 100644 --- a/python/graphscope/gsctl/scripts/install_deps_command.sh +++ b/python/graphscope/gsctl/scripts/install_deps_command.sh @@ -234,14 +234,18 @@ install_basic_packages_universal() { elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then if [[ "${OS_VERSION}" -eq "7" ]]; then ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} + # change the source for centos-release-scl-rh + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} else if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then ${SUDO} yum install -y 'dnf-command(config-manager)' ${SUDO} dnf install -y epel-release --allowerasing else - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* ${SUDO} yum install -y 'dnf-command(config-manager)' ${SUDO} dnf install -y epel-release ${SUDO} dnf config-manager --set-enabled powertools @@ -324,7 +328,12 @@ install_dependencies_analytical_universal() { } install_interactive_deps() { - install_hiactor "${install_prefix}" + # seastar can not be built on macos and centos7 + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + install_hiactor "${install_prefix}" + else + warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." + fi } write_env_config() { @@ -414,12 +423,7 @@ install_deps_for_dev() { install_rust_universal install_cppkafka "${deps_prefix}" "${install_prefix}" # install dependencies for flex interactive - # can not install on macos since seastar can not be built on macos - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - warning "Skip installing dependencies for flex interactive on macOS." - else - install_interactive_deps - fi + install_interactive_deps fi write_env_config diff --git a/python/setup.py b/python/setup.py index 6910fbf4d35e..25824fa68319 100644 --- a/python/setup.py +++ b/python/setup.py @@ -251,19 +251,20 @@ def run(self): graphlearn_shared_lib = "libgraphlearn_shared.dylib" else: graphlearn_shared_lib = "libgraphlearn_shared.so" - if not os.path.isfile( - os.path.join( - pkg_root, - "..", - "learning_engine", - "graph-learn", - "graphlearn", - "built", - "lib", - graphlearn_shared_lib, - ) - ): - raise ValueError("You must build the graphlearn library at first") + if os.environ.get("WITHOUT_LEARNING_ENGINE", None) is None: + if not os.path.isfile( + os.path.join( + pkg_root, + "..", + "learning_engine", + "graph-learn", + "graphlearn", + "built", + "lib", + graphlearn_shared_lib, + ) + ): + raise ValueError("You must build the graphlearn library at first") self.run_command("build_proto") bdist_wheel.run(self) @@ -313,6 +314,9 @@ def parsed_package_data(): def build_learning_engine(): + if os.environ.get("WITHOUT_LEARNING_ENGINE", None) is not None: + return [] + ext_modules = [graphlearn_ext()] if torch and os.path.exists(os.path.join(glt_root_path, "graphlearn_torch")): sys.path.insert(