diff --git a/.github/free-disk-space.sh b/.github/free-disk-space.sh index 52d311dc7385..9260e8ee1dcd 100755 --- a/.github/free-disk-space.sh +++ b/.github/free-disk-space.sh @@ -47,5 +47,4 @@ rm -rf /usr/share/dotnet/ sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc -sudo rm -rf /opt/hostedtoolcache/CodeQL df -h diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml index c47810203b8a..22eba205b978 100644 --- a/.github/workflows/build-graphscope-dev-images.yml +++ b/.github/workflows/build-graphscope-dev-images.yml @@ -1,10 +1,9 @@ -name: Build GraphScope Development and Wheel Images +name: Build GraphScope Development Images # build images for: -# 1) wheel: including all dependencies for graphscope's wheel package. -# 2) graphscope-dev: including all dependencies for graphscope's development env. -# 3) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. -# 4) vineyard-runtime: including all vineyard-related running dependencies. +# 1) graphscope-dev: including all dependencies for graphscope development env. +# 2) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine. +# 3) vineyard-runtime: including all vineyard-related running dependencies. # Note that: # Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github. on: @@ -14,11 +13,6 @@ on: description: 'Version for Vineyard (v6d)' required: true default: 'main' - build_wheel: - description: 'Whether to build dev-wheel image' - required: true - default: true - type: boolean build_graphscope_dev: description: 'Whether to build graphscope-dev image' required: true @@ -34,6 +28,7 @@ on: - main paths: - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-dev-images.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -43,116 +38,9 @@ env: REGISTRY: registry.cn-hongkong.aliyuncs.com jobs: - build-wheel-image-x86-64: - runs-on: ubuntu-20.04 - if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request') - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--x86_64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - build-wheel-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request') - if: false - - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - # build wheel image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s - VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} - if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:wheel--aarch64 - make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} - else - # pull_request: use default vineyard_version - make dev-wheel - fi - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # dev-wheel image - sudo docker tag graphscope/graphscope-dev:wheel-${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:wheel-${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} || true - - manifest-push-wheel-image: - runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_wheel == 'true' }} - if: false - - needs: [build-wheel-image-x86-64, build-wheel-image-aarch64] - steps: - - name: Create and Push Docker Manifest - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # manifest create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-aarch64 - # manifest push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} - - build-graphscope-dev-image-x86-64: - runs-on: ubuntu-20.04 + build-graphscope-dev-image-amd64: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') - + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -160,11 +48,11 @@ jobs: - name: Build Image run: | - # build graphscope dev image with specified v6d's version + # build graphscope dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-x86_64 + # graphscope/graphscope-dev:-amd64 make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -178,19 +66,13 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 - build-graphscope-dev-image-aarch64: + build-graphscope-dev-image-arm64: runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') if: false - steps: - uses: actions/checkout@v4 with: @@ -198,11 +80,11 @@ jobs: - name: Build Image run: | - # build graphscope dev image with specified v6d's version + # build graphscope dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/graphscope-dev:-aarch64 + # graphscope/graphscope-dev:-arm64 make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -216,30 +98,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} # graphscope-dev image - sudo docker tag graphscope/graphscope-dev:${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/graphscope-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} || true + sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 manifest-push-graphscope-dev-image: runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_graphscope_dev == 'true' }} if: false - - needs: [build-graphscope-dev-image-x86-64, build-graphscope-dev-image-aarch64] + needs: [build-graphscope-dev-image-amd64, build-graphscope-dev-image-arm64] steps: - name: Create and Push Docker Manifest env: @@ -250,15 +117,14 @@ jobs: # manifest create sudo docker manifest create \ ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-aarch64 + ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-amd64 \ + ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-arm64 # manifest push sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} - build-vineyard-dev-image-x86-64: - runs-on: ubuntu-20.04 + build-vineyard-dev-image-amd64: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') - + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 with: @@ -266,11 +132,11 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-dev:-x86_64 + # graphscope/vineyard-dev:-amd64 make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -284,19 +150,13 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 - build-vineyard-dev-image-aarch64: + build-vineyard-dev-image-arm64: runs-on: [self-hosted, Linux, ARM64] - # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') if: false - steps: - uses: actions/checkout@v4 with: @@ -304,7 +164,7 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then @@ -322,30 +182,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} # vineyard-dev image - sudo docker tag graphscope/vineyard-dev:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }}-${arch} - # clean - sudo docker rmi -f graphscope/vineyard-dev:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} || true + sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 manifest-push-vineyard-dev-image: runs-on: ubuntu-20.04 - # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} if: false - - needs: [build-vineyard-dev-image-x86-64, build-vineyard-dev-image-aarch64] + needs: [build-vineyard-dev-image-amd64, build-vineyard-dev-image-arm64] steps: - name: Create and Push Docker Manifest env: @@ -356,18 +201,16 @@ jobs: # manifest create sudo docker manifest create \ ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-x86_64 \ - ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-aarch64 + ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-amd64 \ + ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-arm64 # manifest push sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} - build-vineyard-runtime-image-x86-64: - runs-on: ubuntu-20.04 - # only trigger this step in 'workflow_dispatch' event, - # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' + build-vineyard-runtime-image-amd64: + # only trigger this step in 'workflow_dispatch' event, since the 'vineyard-dev' image isn't actually pushed in 'pull_request' if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request') - - needs: [build-vineyard-dev-image-x86-64] + runs-on: ubuntu-20.04 + needs: [build-vineyard-dev-image-amd64] steps: - uses: actions/checkout@v4 with: @@ -375,11 +218,11 @@ jobs: - name: Build Image run: | - # build vineyard runtime image with specified v6d's version - cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + # build vineyard runtime image with specified v6d version + cd ${GITHUB_WORKSPACE}/k8s if [[ -n ${VINEYARD_VERSION} ]]; then - # graphscope/vineyard-runtime:-x86_64 + # graphscope/vineyard-runtime:-amd64 make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION} else # pull_request: use default vineyard_version @@ -393,15 +236,11 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=${{ github.event.inputs.v6d_version }} - # vineyard-runtime image - sudo docker tag graphscope/vineyard-runtime:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 - build-vineyard-runtime-image-aarch64: + build-vineyard-runtime-image-arm64: runs-on: [self-hosted, Linux, ARM64] # only trigger this step in 'workflow_dispatch' event, # since the 'vineyard-dev' image isn't actually pushed in 'pull_request' @@ -416,7 +255,7 @@ jobs: - name: Build Image run: | - # build vineyard dev image with specified v6d's version + # build vineyard dev image with specified v6d version cd ${GITHUB_WORKSPACE}/k8s VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} if [[ -n ${VINEYARD_VERSION} ]]; then @@ -456,7 +295,7 @@ jobs: # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} if: false - needs: [build-vineyard-runtime-image-x86-64, build-vineyard-runtime-image-aarch64] + needs: [build-vineyard-runtime-image-amd64, build-vineyard-runtime-image-arm64] steps: - name: Create and Push Docker Manifest env: diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml new file mode 100644 index 000000000000..791a88e4a1fa --- /dev/null +++ b/.github/workflows/build-graphscope-dev-wheel-images.yml @@ -0,0 +1,56 @@ +name: Build GraphScope Wheel Images + +# build `graphscope/graphscope-dev:wheel-{v6d_version}` image based on manylinux, +# including all dependencies for building graphscope wheel package. +on: + workflow_dispatch: + inputs: + v6d_version: + description: 'Version for Vineyard (v6d)' + required: true + default: 'main' + pull_request: + branches: + - main + paths: + - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-dev-wheel-images.yml' + +concurrency: + group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +env: + REGISTRY: registry.cn-hongkong.aliyuncs.com + +jobs: + build-wheel-image-amd64: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build Image + run: | + # build wheel image with specified v6d's version + cd ${GITHUB_WORKSPACE}/k8s + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + if [[ -n ${VINEYARD_VERSION} ]]; then + # graphscope/graphscope-dev:wheel--amd64 + make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION} + else + # pull_request: use default vineyard_version + make dev-wheel + fi + + - name: Release Image + if: ${{ github.event_name == 'workflow_dispatch' }} + env: + docker_password: ${{ secrets.DOCKER_PASSWORD }} + docker_username: ${{ secrets.DOCKER_USER }} + run: | + echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin + VINEYARD_VERSION=${{ github.event.inputs.v6d_version }} + sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml deleted file mode 100644 index 0c2844ab23ae..000000000000 --- a/.github/workflows/build-graphscope-manylinux-ext-images.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: Build GraphScope Manylinux-Ext Images - -# build graphscope-dev-base image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. -on: - workflow_dispatch: - pull_request: - branches: - - main - paths: - - 'python/graphscope/gsctl/scripts/**' - -concurrency: - group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} - cancel-in-progress: true - -env: - REGISTRY: registry.cn-hongkong.aliyuncs.com - -jobs: - build-manylinux-ext-image-x86-64: - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux:ext-x86_64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # x86_64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext - - build-manylinux-ext-image-aarch64: - runs-on: [self-hosted, Linux, ARM64] - if: false - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build Image - run: | - cd ${GITHUB_WORKSPACE}/k8s - # output: graphscope/manylinux2014:ext-aarch64 - make manylinux2014-ext - - - name: Release Image - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # manylinux2014 image - sudo docker tag graphscope/manylinux2014:${tag} ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} - - - name: Clean Image - run: | - # aarch64 - arch=$(uname -m) - # image tag - tag=ext-${arch} - # clean - sudo docker rmi -f graphscope/manylinux2014:${tag} || true - sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/manylinux2014:${tag} || true - - manifest_push_manylinux-ext-image: - runs-on: ubuntu-20.04 - if: false - needs: [build-manylinux-ext-image-x86-64, build-manylinux-ext-image-aarch64] - steps: - - name: Create and Push Docker Manifest - if: ${{ github.event_name == 'workflow_dispatch' }} - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin - # create - sudo docker manifest create \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-x86_64 \ - ${{ env.REGISTRY }}/graphscope/manylinux2014:ext-aarch64 - # push - sudo docker manifest push ${{ env.REGISTRY }}/graphscope/manylinux2014:ext diff --git a/.github/workflows/build-graphscope-manylinux-images.yml b/.github/workflows/build-graphscope-manylinux-images.yml new file mode 100644 index 000000000000..123dc1213335 --- /dev/null +++ b/.github/workflows/build-graphscope-manylinux-images.yml @@ -0,0 +1,45 @@ +name: Build GraphScope Manylinux Images + +# build `graphscope/manylinux2014` image based on centos7, including all necessary +# dependencies except vineyard +on: + workflow_dispatch: + pull_request: + branches: + - main + paths: + - 'python/graphscope/gsctl/scripts/**' + - '.github/workflows/build-graphscope-manylinux-images.yml' + + +concurrency: + group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +env: + REGISTRY: registry.cn-hongkong.aliyuncs.com + +jobs: + build-manylinux-image-amd64: + runs-on: ubuntu-20.04 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build Image + run: | + cd ${GITHUB_WORKSPACE}/k8s + # output: graphscope/manylinux:amd64 + make manylinux2014 + + - name: Release Image + if: ${{ github.event_name == 'workflow_dispatch' }} + env: + docker_password: ${{ secrets.DOCKER_PASSWORD }} + docker_username: ${{ secrets.DOCKER_USER }} + run: | + echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin + # manylinux2014 image + sudo docker tag graphscope/manylinux2014:amd64 ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64 + sudo docker push ${{ env.REGISTRY }}/graphscope/manylinux2014:amd64 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000000..d901aa384d85 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,121 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + push: + branches: [ "main" ] + paths: + - 'analytical_engine/**' + - 'interactive_engine/**' + - 'learning_engine/**' + - 'coordinator/**' + - 'flex/**' + - 'python/**' + # Commented out to save time; reopen after install-deps refactored. + # pull_request: + # branches: [ "main" ] + schedule: + - cron: '38 20 * * 5' + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners (GitHub.com only) + # Consider using larger runners or machines with greater resources for possible analysis time improvements. + runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} + timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} + permissions: + # required for all workflows + security-events: write + + # required to fetch internal or private CodeQL packs + packages: read + + # only required for workflows in private repositories + actions: read + contents: read + + strategy: + fail-fast: false + matrix: + include: + - language: c-cpp + build-mode: manual + - language: java-kotlin + build-mode: none # This mode only analyzes Java. Set this to 'autobuild' or 'manual' to analyze Kotlin too. + - language: javascript-typescript + build-mode: none + - language: python + build-mode: none + # CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' + # Use `c-cpp` to analyze code written in C, C++ or both + # Use 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both + # To learn more about changing the languages that are analyzed or customizing the build mode for your analysis, + # see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning. + # If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how + # your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + build-mode: ${{ matrix.build-mode }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + # If the analyze step fails for one of the languages you are analyzing with + # "We were unable to automatically build your code", modify the matrix above + # to set the build mode to "manual" for that language. Then modify this step + # to build your code. + # ℹ️ Command-line programs to run using the OS shell. + # πŸ“š See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + - if: matrix.build-mode == 'manual' + name: Cpp Manual Build and Analysis + shell: bash + run: | + echo 'Install dependencies' + pip install gsctl + gsctl install-deps dev + source ~/.graphscope_env + echo 'Making GRAPE' + cd analytical_engine + mkdir build + cd build + cmake -DBUILD_TESTS=OFF \ + -DENABLE_JAVA_SDK=OFF .. + make + echo 'Clean extra artifacts to free device space.' + du -hcd1 ~ /opt/graphscope || true + rm -rf /opt/vineyard/bin/run_* || true + rm -rf ~/.local || true + rm -rf ~/.rustup || true + # PWD: /analytical_engine/build + bash ../../.github/free-disk-space.sh || true + du -hcd1 ~ /opt/graphscope || true + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 605fd4cb2605..8b02f4a002cd 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,155 +1,20 @@ -name: Docs +name: Docs Deployment on: - workflow_dispatch: push: branches: - main - - docs - - dev/docs tags: - "v*" - pull_request: - branches: - - main - - docs - - dev/docs - jobs: build: runs-on: ubuntu-20.04 - permissions: - issues: write - pull-requests: write - contents: write steps: - name: Checkout Code uses: actions/checkout@v4 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} submodules: true - fetch-depth: 0 - - - uses: dorny/paths-filter@v2 - id: changes - with: - filters: | - src: - - 'docs/**' - - - name: Setup Java11 - uses: actions/setup-java@v3 - with: - distribution: 'zulu' - java-version: '11' - - - uses: actions/setup-node@v3 - with: - node-version: 16 - - - name: Leave a marker - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' }} - run: | - touch ${GITHUB_WORKSPACE}/preview-the-docs.mark - - - name: Leave the comment on pull request when started - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - ⚑️ Deploying PR Preview ${{ github.event.pull_request.head.sha }} to [surge.sh](https://alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh) ... [Build logs](https://github.com/alibaba/GraphScope/runs/${{ github.run_id }}) - - - - - - πŸ€– By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Cpp Format and Lint Check - run: | - # install clang-format - sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format - sudo chmod +x /usr/bin/clang-format - - # run format - cd analytical_engine/ - find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.h" | xargs clang-format -i --style=file - find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.cc" | xargs clang-format -i --style=file - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| clang-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo "| make gsa_clformat" - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with clang-format-8, which can be obtained from" - echo "|" - echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Java Format and Lint Check - run: | - wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar - - files_to_format=$(git ls-files *.java) - - # run formatter in-place - java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format - - # validate format - function prepend() { while read line; do echo "${1}${line}"; done; } - - GIT_DIFF=$(git diff --ignore-submodules) - if [[ -n $GIT_DIFF ]]; then - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - echo "| google-java-format failures found!" - echo "|" - echo "$GIT_DIFF" | prepend "| " - echo "|" - echo "| Run: " - echo "|" - echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' - echo "|" - echo "| to fix this error." - echo "|" - echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" - echo "|" - echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" - echo "|" - echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" - exit -1 - fi - - - name: Python Format and Lint Check - run: | - echo "Checking formatting for $GITHUB_REPOSITORY" - pip3 install -r coordinator/requirements-dev.txt - pip3 install -r python/requirements-dev.txt - pushd python - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . - popd - pushd coordinator - python3 -m isort --check --diff . - python3 -m black --check --diff . - python3 -m flake8 . + fetch-depth: 1 - name: Generate Docs shell: bash @@ -170,61 +35,11 @@ jobs: make graphscope-docs fi - - name: Preview using surge - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - run: | - npm install -g surge - surge ./docs/_build/latest/html \ - alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh \ - --token ${{ secrets.SURGE_TOKEN }} - - - name: Leave the comment on pull request when succeed - if: ${{ steps.changes.outputs.src == 'true' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - 🎊 PR Preview ${{ github.event.pull_request.head.sha }} has been successfully built and deployed to https://alibaba-graphscope-build-pr-${{ github.event.number }}.surge.sh - - - - - - πŸ€– By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Check file existence - id: check_files - if: ${{ failure() }} - uses: andstor/file-existence-action@v2 - with: - files: "preview-the-docs.mark" - - - name: Leave the comment on pull request when failed - if: ${{ failure() && steps.check_files.outputs.files_exists == 'true' && github.event.pull_request.head.repo.full_name == 'alibaba/GraphScope' }} - uses: actions-cool/maintain-one-comment@v3 - with: - token: ${{ secrets.GITHUB_TOKEN }} - body: | - 😭 Deploy PR Preview ${{ github.event.pull_request.head.sha }} failed. [Build logs](https://github.com/alibaba/GraphScope/runs/${{ github.run_id }}) - - - - - - πŸ€– By [surge-preview](https://surge.sh/) - body-include: '' - - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - if: false - - name: Upload Docs - if: ${{ github.event_name == 'push' && github.repository == 'alibaba/GraphScope' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) }} + if: ${{ github.repository == 'alibaba/GraphScope' }} shell: bash run: | shopt -s extglob - rm google-java-format-1.13.0-all-deps.jar* || true git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" diff --git a/.github/workflows/flex-interactive.yml b/.github/workflows/flex-interactive.yml index 18555bb44fb7..37ca3a52fd53 100644 --- a/.github/workflows/flex-interactive.yml +++ b/.github/workflows/flex-interactive.yml @@ -33,11 +33,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - name: Build Image run: | cd ${GITHUB_WORKSPACE} @@ -59,7 +54,7 @@ jobs: # install gsctl python3 -m pip install ${GITHUB_WORKSPACE}/python/dist/*.whl # launch service: 8080 for coordinator http port; 7687 for cypher port; - docker run -p 8080:8080 -p 7688:7687 registry.cn-hongkong.aliyuncs.com/graphscope/interactive:${SHORT_SHA}-x86_64 --enable-coordinator & + docker run -p 8080:8080 -p 7688:7687 graphscope/interactive:latest --enable-coordinator & sleep 20 # test python3 -m pip install --no-cache-dir pytest pytest-xdist diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 97f666d60dcb..9481af75685f 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -252,6 +252,7 @@ jobs: #set back sed -i 's/default_graph: graph_algo/default_graph: ldbc/g' ../tests/hqps/engine_config_test.yaml + rm -rf /tmp/codegen - name: Test cypher&cpp procedure generation and loading env: diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index fa6baad55298..0c096deb65a9 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -1,90 +1,247 @@ -name: PR-Check +name: PR Check on: - pull_request: - types: - - opened - - reopened - - edited - - synchronize + pull_request_target: branches: - main jobs: - conventional-pr-check: + PR-Check: runs-on: ubuntu-20.04 + permissions: + pull-requests: write + steps: + - name: Conventional PR Check + uses: amannn/action-semantic-pull-request@v5 + id: pr-convention + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + # Types allowed (newline-delimited). + # Default: https://github.com/commitizen/conventional-commit-types + types: | + build + ci + docs + feat + fix + perf + refactor + test + chore + # Scopes allowed (newline-delimited). + scopes: | + core + python + k8s + coordinator + one + interactive + insight + analytical + learning + flex + # A scope can be not provided. + requireScope: false + disallowScopes: | + release + [A-Z]+ + # If the PR contains one of these newline-delimited labels, the + # validation is skipped. + ignoreLabels: | + bot + ignore-semantic-pull-request + + - name: Comments if PR Title is not conventional + id: lint_pr_title + uses: marocchino/sticky-pull-request-comment@v2 + # When the previous steps fails, the workflow would stop. By adding this + # condition you can continue the execution with the populated error message. + if: always() && (steps.pr-convention.outputs.error_message != null) + with: + header: pr-title-lint-error + message: | + Hey there and thank you for opening this pull request! πŸ‘‹πŸΌ + + We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) \ + and it looks like your proposed title needs to be adjusted. + + Details: + ``` + ${{ steps.pr-convention.outputs.error_message }} + ``` + + + # Delete a previous comment when the issue has been resolved + - name: Delete Comment if PR Title is conventional + if: ${{ steps.lint_pr_title.outputs.error_message == null }} + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: pr-title-lint-error + delete: true + - name: Checkout Code uses: actions/checkout@v4 with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} + ref: ${{ github.event.pull_request.head.sha }} submodules: true - fetch-depth: 0 + fetch-depth: 1 - - uses: amannn/action-semantic-pull-request@v5 - id: pr-convention - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Java11 + uses: actions/setup-java@v3 with: - # Types allowed (newline-delimited). - # Default: https://github.com/commitizen/conventional-commit-types - types: | - build - ci - docs - feat - fix - perf - refactor - test - chore - # Scopes allowed (newline-delimited). - scopes: | - core - python - k8s - coordinator - one - interactive - insight - analytical - learning - flex - # A scope can be not provided. - requireScope: false - disallowScopes: | - release - [A-Z]+ - # If the PR contains one of these newline-delimited labels, the - # validation is skipped. - ignoreLabels: | - bot - ignore-semantic-pull-request - - feature-docs-check: - runs-on: ubuntu-20.04 - needs: conventional-pr-check - steps: - - uses: dorny/paths-filter@v2 - id: doc-changes + distribution: 'zulu' + java-version: '11' + + - name: Get PR Changes + uses: dorny/paths-filter@v3 + id: changes with: filters: | - src: - - 'docs/**' + docs: + - 'docs/**' - - uses: actions-ecosystem/action-regex-match@v2 - id: pr-regex-match - with: - text: ${{ github.event.pull_request.title }} - regex: 'feat.*|refactor.*' + - name: Cpp Format and Lint Check + run: | + # install clang-format + sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-22538c65/clang-format-8_linux-amd64 --output /usr/bin/clang-format + sudo chmod +x /usr/bin/clang-format + + # run format + cd analytical_engine/ + find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.h" | xargs clang-format -i --style=file + find ./apps ./benchmarks ./core ./frame ./misc ./test -name "*.cc" | xargs clang-format -i --style=file + + # validate format + function prepend() { while read line; do echo "${1}${line}"; done; } - - if: ${{ steps.pr-regex-match.outputs.match != '' && steps.doc-changes.outputs.src == 'false' }} + GIT_DIFF=$(git diff --ignore-submodules) + if [[ -n $GIT_DIFF ]]; then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "| clang-format failures found!" + echo "|" + echo "$GIT_DIFF" | prepend "| " + echo "|" + echo "| Run: " + echo "|" + echo "| make gsa_clformat" + echo "|" + echo "| to fix this error." + echo "|" + echo "| Ensure you are working with clang-format-8, which can be obtained from" + echo "|" + echo "| https://github.com/muttleyxd/clang-tools-static-binaries/releases" + echo "|" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + exit -1 + fi + + - name: Java Format and Lint Check + run: | + wget https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar + + files_to_format=$(git ls-files *.java) + + # run formatter in-place + java -jar ${GITHUB_WORKSPACE}/google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $files_to_format + + # validate format + function prepend() { while read line; do echo "${1}${line}"; done; } + + GIT_DIFF=$(git diff --ignore-submodules) + if [[ -n $GIT_DIFF ]]; then + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + echo "| google-java-format failures found!" + echo "|" + echo "$GIT_DIFF" | prepend "| " + echo "|" + echo "| Run: " + echo "|" + echo '| java -jar google-java-format-1.13.0-all-deps.jar --aosp --skip-javadoc-formatting -i $(git ls-files **/*.java)' + echo "|" + echo "| to fix this error." + echo "|" + echo "| Ensure you are working with google-java-format-1.13.0, which can be obtained from" + echo "|" + echo "| https://github.com/google/google-java-format/releases/download/v1.13.0/google-java-format-1.13.0-all-deps.jar" + echo "|" + echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~" + exit -1 + fi + + - name: Python Format and Lint Check run: | - # echo "title=${{ github.event.pull_request.title }}" - # echo "steps.pr-regex-match.outputs.match=${{ steps.pr-regex-match.outputs.match }}" - # echo "steps.doc-changes.outputs.src=${{ steps.doc-changes.outputs.src }}" - echo " ❌ Uh oh! ❌ \n - We suggest that a PR with type @feat should has corresponding documentations. \n - If you believe this PR could be merged without documentation, please add @yecol as an extra reviewer for confirmation." - exit 1 + echo "Checking formatting for $GITHUB_REPOSITORY" + pip3 install -r coordinator/requirements-dev.txt + pip3 install -r python/requirements-dev.txt + pushd python + python3 -m isort --check --diff . + python3 -m black --check --diff . + python3 -m flake8 . + popd + pushd coordinator + python3 -m isort --check --diff . + python3 -m black --check --diff . + python3 -m flake8 . + + - name: Generate Docs + shell: bash + run: | + # Install pip dependencies, build builtin gar, and generate proto stuffs. + sudo apt update + sudo apt install -y doxygen graphviz + + # generate a tagged version + cd ${GITHUB_WORKSPACE} + make graphscope-docs + + # generate a stable version + tag=$(git describe --exact-match --tags HEAD 2>/dev/null || true) + if [ ! -z "$tag" ]; + then + export TAG_VER=stable + make graphscope-docs + fi + # Preview on comment will be attached by Cloudflare if files in /docs changed. + + - name: Preview on Cloudflare + id: preview + uses: cloudflare/pages-action@v1 + if: ${{ steps.changes.outputs.docs }} + with: + apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} + accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} + projectName: graphscope-docs-preview + directory: docs/_build/latest/html + + ########################################################################### + # Steps to give feedbacks by commentting PR + ########################################################################### + + - name: Comments if Docs not present/changed while required + uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ ( github.event.pull_request.title == 'feat.*' || github.event.pull_request.title == 'refactor.*' ) && steps.doc-changes.outputs.docs == 'false' }} + with: + header: pr-docs-change-required + message: | + ❌ Uh oh! ❌ + We suggest that a PR with type feat/refactor should be well documented. + If you believe this PR could be merged without documentation, please add @yecol as an extra reviewer for confirmation. + + # Delete a previous comment when the issue has been resolved + - name: Delete Comment if Docs changes committed + if: ${{ steps.changes.outputs.docs}} + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: pr-docs-change-required + delete: true + + - name: Comments with the PR Preview URL + uses: marocchino/sticky-pull-request-comment@v2 + if: ${{ steps.changes.outputs.docs}} + with: + header: pr-preview-url + message: | + Please check the preview of the documentation changes at + [${{ steps.preview.outputs.url }}](${{ steps.preview.outputs.url }}) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77adb6251a79..df8b4d74887e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,23 +64,17 @@ jobs: password: ${{ secrets.PYPI_PASSWORD }} packages_dir: upload_pypi/ - build-interactive-image-x86-64: + build-interactive-image-amd64: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') - runs-on: ubuntu-20.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - - name: Add envs to GITHUB_ENV - run: | - short_sha=$(git rev-parse --short HEAD) - echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV - - name: Build Interactive Image run: | - cd ${GITHUB_WORKSPACE}/python - python3 -m pip install --upgrade pip && python3 -m pip install -r requirements.txt && python3 setup.py build_proto cd ${GITHUB_WORKSPACE} + python3 -m pip install --upgrade pip && python3 -m pip install click python3 ./gsctl.py flexbuild interactive --app docker - name: Extract Tag Name @@ -95,10 +89,10 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - sudo docker tag ${{ env.INTERACTIVE_IMAGE }}:${SHORT_SHA}-x86_64 ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }} - sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }} + sudo docker tag graphscope/interactive:latest ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 + sudo docker push ${{ env.INTERACTIVE_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 - build-gss-image-x86-64: + build-gss-image-amd64: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') runs-on: ubuntu-20.04 steps: @@ -114,25 +108,6 @@ jobs: cd ${GITHUB_WORKSPACE}/k8s sudo make graphscope-store VERSION=${SHORT_SHA} - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # x86_64 - # arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}" - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - name: Extract Tag Name if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} id: tag @@ -145,12 +120,10 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # x86_64 - # arch=$(uname -m) sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} - build-gss-image-aarch64: + build-gss-image-arm64: # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') if: false runs-on: [self-hosted, Linux, ARM64] @@ -167,25 +140,6 @@ jobs: cd ${GITHUB_WORKSPACE}/k8s sudo make graphscope-store VERSION=${SHORT_SHA} - - name: Release Nightly Image - # if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }} - # GraphScope-Store doesn't need nightly release yet. - # To save some resources, comment this step out temporarily. - if: false - env: - docker_password: ${{ secrets.DOCKER_PASSWORD }} - docker_username: ${{ secrets.DOCKER_USER }} - run: | - echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${tag} - sudo docker push ${{ env.GSS_IMAGE }}:${tag} - - name: Extract Tag Name if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} id: tag @@ -198,38 +152,15 @@ jobs: docker_username: ${{ secrets.DOCKER_USER }} run: | echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin - # aarch64 - arch=$(uname -m) - sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} - - - name: Clean Nightly Image - if: false - run: | - # docker tag: 0.15.0 -> 0.15.0a20220808 - # aarch64 - arch=$(uname -m) - time=$(date "+%Y%m%d") - version=$(cat ${GITHUB_WORKSPACE}/VERSION) - tag="${version}a${time}"-${arch} - # clean - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${tag} || true - - - name: Clean Release Image - if: always() - run: | - # aarch64 - arch=$(uname -m) - sudo docker rmi -f graphscope-store:${SHORT_SHA} || true - sudo docker rmi -f ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-${arch} || true + sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 + sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 push-gss-image-manifest: # if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} if: false runs-on: ubuntu-20.04 - needs: [build-gss-image-x86-64, build-gss-image-aarch64] + needs: [build-gss-image-amd64, build-gss-image-arm64] steps: - uses: actions/checkout@v4 @@ -251,8 +182,8 @@ jobs: # create sudo docker manifest create \ ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-x86_64 \ - ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-aarch64 + ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-amd64 \ + ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}-arm64 # push sudo docker manifest push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }} diff --git a/analytical_engine/apps/flash/flash_context.h b/analytical_engine/apps/flash/flash_context.h index 8fbb13ddc4a0..1c1a6b77bd6b 100644 --- a/analytical_engine/apps/flash/flash_context.h +++ b/analytical_engine/apps/flash/flash_context.h @@ -19,9 +19,11 @@ limitations under the License. #include #include -#include "flash/flash_ware.h" #include "grape/grape.h" +#include "core/context/tensor_context.h" +#include "flash/flash_ware.h" + namespace gs { /** diff --git a/analytical_engine/apps/flash/flash_ware.h b/analytical_engine/apps/flash/flash_ware.h index 7e5b920dba36..b4113f5c4c88 100644 --- a/analytical_engine/apps/flash/flash_ware.h +++ b/analytical_engine/apps/flash/flash_ware.h @@ -26,6 +26,7 @@ limitations under the License. #include "grape/parallel/parallel_message_manager.h" #include "grape/worker/comm_spec.h" +#include "core/config.h" #include "flash/flash_bitset.h" #include "flash/vertex_subset.h" @@ -141,7 +142,7 @@ class FlashWare : public grape::Communicator, public grape::ParallelEngine { vid_t n_; vid_t n_loc_; fid_t pid_; - int n_procs_; + fid_t n_procs_; int n_threads_; std::vector masters_; std::vector mirrors_; diff --git a/charts/graphscope-store/templates/configmap.yaml b/charts/graphscope-store/templates/configmap.yaml index 49c4d008456a..698e2881e626 100644 --- a/charts/graphscope-store/templates/configmap.yaml +++ b/charts/graphscope-store/templates/configmap.yaml @@ -66,6 +66,8 @@ data: graph.physical.opt={{ .Values.graphPhysicalOpt }} gremlin.script.language.name={{ .Values.gremlinScriptLanguageName }} query.execution.timeout.ms={{ .Values.queryExecutionTimeoutMs }} + graph.planner.join.min.pattern.size={{ .Values.graphPlannerJoinMinPatternSize }} + graph.planner.cbo.glogue.size={{ .Values.graphPlannerCboGlogueSize }} log4rs.config=LOG4RS_CONFIG ## Auth config diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml index 8a088ba67a97..ec9d18396486 100644 --- a/charts/graphscope-store/templates/coordinator/statefulset.yaml +++ b/charts/graphscope-store/templates/coordinator/statefulset.yaml @@ -44,6 +44,14 @@ spec: {{- if .Values.coordinator.schedulerName }} schedulerName: {{ .Values.coordinator.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "coordinator" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "coordinator" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml index 4689313603c8..1cba12b122d3 100644 --- a/charts/graphscope-store/templates/frontend/statefulset.yaml +++ b/charts/graphscope-store/templates/frontend/statefulset.yaml @@ -44,6 +44,14 @@ spec: {{- if .Values.frontend.schedulerName }} schedulerName: {{ .Values.frontend.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "frontend" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "frontend" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/templates/onepod/statefulset.yaml b/charts/graphscope-store/templates/onepod/statefulset.yaml index d05c928a0a46..ae27127d0b86 100644 --- a/charts/graphscope-store/templates/onepod/statefulset.yaml +++ b/charts/graphscope-store/templates/onepod/statefulset.yaml @@ -44,6 +44,14 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/templates/portal/statefulset.yaml b/charts/graphscope-store/templates/portal/statefulset.yaml index c1b39d21eff7..1e8c5ee471cb 100644 --- a/charts/graphscope-store/templates/portal/statefulset.yaml +++ b/charts/graphscope-store/templates/portal/statefulset.yaml @@ -44,6 +44,14 @@ spec: {{- if .Values.portal.schedulerName }} schedulerName: {{ .Values.portal.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "portal" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "portal" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/templates/store/statefulset-backup.yaml b/charts/graphscope-store/templates/store/statefulset-backup.yaml index aedefc5b5bb1..73483e840af8 100644 --- a/charts/graphscope-store/templates/store/statefulset-backup.yaml +++ b/charts/graphscope-store/templates/store/statefulset-backup.yaml @@ -48,6 +48,14 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml index 9bb943aa27c9..9ebc14b71d3a 100644 --- a/charts/graphscope-store/templates/store/statefulset.yaml +++ b/charts/graphscope-store/templates/store/statefulset.yaml @@ -44,6 +44,14 @@ spec: {{- if .Values.store.schedulerName }} schedulerName: {{ .Values.store.schedulerName | quote }} {{- end }} + {{- if .Values.affinity }} + affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }} + {{- else }} + affinity: + podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "store" "context" $) | nindent 10 }} + podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "store" "context" $) | nindent 10 }} + nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }} + {{- end }} {{- if .Values.nodeSelector }} nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }} {{- end }} diff --git a/charts/graphscope-store/values.yaml b/charts/graphscope-store/values.yaml index 111389153dfb..f2c94f4fd91e 100644 --- a/charts/graphscope-store/values.yaml +++ b/charts/graphscope-store/values.yaml @@ -538,7 +538,7 @@ storeDataPath: "/var/lib/graphscope-store" storeDataDownloadPath: "/var/lib/graphscope-store/download" storeDataSecondaryPath: "/home/graphscope/secondary" storeWriteThreadCount: 1 -storeQueueBufferSize: 102400 +storeQueueBufferSize: "1024000" storeGcIntervalMs: 5000 @@ -591,6 +591,8 @@ graphPlannerRules: FilterIntoJoinRule, FilterMatchRule, ExtendIntersectRule, Exp gremlinScriptLanguageName: antlr_gremlin_traversal graphPhysicalOpt: ffi queryExecutionTimeoutMs: 600000 +graphPlannerJoinMinPatternSize: 5 +graphPlannerCboGlogueSize: 3 ## Key-value pair separated by ; ## For example extraConfig="k1=v1;k2=v2" diff --git a/docs/deployment/deploy_graphscope_with_helm.md b/docs/deployment/deploy_graphscope_with_helm.md index fa0015ac9084..348bf82215a8 100644 --- a/docs/deployment/deploy_graphscope_with_helm.md +++ b/docs/deployment/deploy_graphscope_with_helm.md @@ -67,6 +67,17 @@ You'll need `graphscope-client` package to `import graphscope`. Since the GraphScope has been running in the cluster, you only need to **connect** to it. Other than the connection procedure, other statements are identical to those GraphScope clusters launched by Python client. +````{tip} +If you meet connection problems with minikube, try getting the service name and exposing the service like this: + +```bash +# kubectl get services +minikube service coordinator-service- --url +``` + +For more usage, you could refer to [minikube's accessing tutorial](https://minikube.sigs.k8s.io/docs/handbook/accessing/). +```` + ```python import graphscope graphscope.set_option(show_log=True) diff --git a/docs/design_of_gae.md b/docs/design_of_gae.md index 16b2f3576a6c..54ebf4072e0a 100644 --- a/docs/design_of_gae.md +++ b/docs/design_of_gae.md @@ -1,4 +1,4 @@ -# Design of GAE +# Design of Analytical Engine In GraphScope, Graph Analytics Engine (GAE) is responsible for handling various graph analytics algorithms. GAE in GraphScope derives from [GRAPE](https://dl.acm.org/doi/10.1145/3282488), a graph processing system proposed on SIGMOD-2017. GRAPE differs from prior systems in its ability to parallelize sequential graph algorithms as a whole. Different from other parallel graph processing systems which need to recast the entire algorithm into a new model, in GRAPE, sequential algorithms can be easily β€œplugged into” with only minor changes and get parallelized to handle large graphs efficiently. In addition to the ease of programming, GRAPE is designed to be highly efficient and flexible, to cope the scale, variety and complexity from real-life graph applications. diff --git a/docs/images/gopt/design_of_gopt.png b/docs/images/gopt/design_of_gopt.png new file mode 100644 index 000000000000..baa608cbb9fc Binary files /dev/null and b/docs/images/gopt/design_of_gopt.png differ diff --git a/docs/images/gopt/movie_data.png b/docs/images/gopt/movie_data.png new file mode 100644 index 000000000000..b00f21bbb029 Binary files /dev/null and b/docs/images/gopt/movie_data.png differ diff --git a/docs/images/gopt/movie_schema.png b/docs/images/gopt/movie_schema.png new file mode 100644 index 000000000000..f54f02d8c368 Binary files /dev/null and b/docs/images/gopt/movie_schema.png differ diff --git a/docs/images/gopt/square_pattern_profile.png b/docs/images/gopt/square_pattern_profile.png new file mode 100644 index 000000000000..7c00c813cf0b Binary files /dev/null and b/docs/images/gopt/square_pattern_profile.png differ diff --git a/docs/images/gopt/square_pattern_with_filter_profile.png b/docs/images/gopt/square_pattern_with_filter_profile.png new file mode 100644 index 000000000000..13e92f7d94d0 Binary files /dev/null and b/docs/images/gopt/square_pattern_with_filter_profile.png differ diff --git a/docs/images/gopt/st_path_profile.png b/docs/images/gopt/st_path_profile.png new file mode 100644 index 000000000000..5571a7f13ba5 Binary files /dev/null and b/docs/images/gopt/st_path_profile.png differ diff --git a/docs/images/gopt/system_overview.png b/docs/images/gopt/system_overview.png new file mode 100644 index 000000000000..110d4c0c385b Binary files /dev/null and b/docs/images/gopt/system_overview.png differ diff --git a/docs/images/gopt/triangle_pattern_profile.png b/docs/images/gopt/triangle_pattern_profile.png new file mode 100644 index 000000000000..3724d67e3866 Binary files /dev/null and b/docs/images/gopt/triangle_pattern_profile.png differ diff --git a/docs/images/gopt/type_error_profile.png b/docs/images/gopt/type_error_profile.png new file mode 100644 index 000000000000..5fcffac0ab37 Binary files /dev/null and b/docs/images/gopt/type_error_profile.png differ diff --git a/docs/index.rst b/docs/index.rst index 0ffd847465d0..5d880d62f575 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -73,7 +73,7 @@ and the vineyard store that offers efficient in-memory data transfers. interactive_engine/deployment interactive_engine/tinkerpop_eco interactive_engine/neo4j_eco - interactive_engine/optimizer + interactive_engine/gopt .. interactive_engine/guide_and_examples interactive_engine/design_of_gie .. interactive_engine/supported_gremlin_steps diff --git a/docs/interactive_engine/gopt.md b/docs/interactive_engine/gopt.md new file mode 100644 index 000000000000..391b76559f18 --- /dev/null +++ b/docs/interactive_engine/gopt.md @@ -0,0 +1,352 @@ +# GOpt: A Graph-Native Query Optimization Framework +## Introducing GOpt +GOpt is a graph-native query optimizer designed to accelerate query execution. It excels in handling hybrid scenarios that combine complex graph pattern matching with relational operations on large graphs. GOpt is not aware of the underlying storage data and focuses solely on computation on top of the data, which makes it easy and fast to be integrated into other graph or relational databases. + +### Core Features + +1. **Property Graph Modeling**: Automatically infers and completes user-provided ambiguous queries for more accurate optimization. +2. **Language Support**: Supports standard [Gremlin](https://tinkerpop.apache.org/gremlin.html) and [Cypher](https://neo4j.com/docs/cypher-manual/current/introduction/) languages, with upcoming [GQL](https://www.gqlstandards.org/) support. +3. **Graph-Native Algorithms**: Implements novel and efficient algorithms for complex graph pattern matching queries. +4. **Lightweight, Serverless Integration**: Seamlessly integrates with databases of various storage formats. + +:::{figure-md} + +system_overview + +GOpt System Overview +::: + +### Why GOpt + +1. **High Performance** + + GOpt is designed and implemented based on years of academic research, with key techniques published in prestigious systems conferences. Our experiments, as documented in our [papers](https://arxiv.org/abs/2401.17786), demonstrate that GOpt outperforms most graph and relational databases in both standard ([LDBC](https://ldbcouncil.org/)) and real-world (Alibaba) graph workloads. + +2. **User-Friendly Interface** + + GOpt offers different layers of SDK tailored to various user requirements. It provides Cypher and Gremlin language support to lower the barrier of entry. User-provided Cypher or Gremlin queries can be more flexible and ambiguous, with GOpt automatically validating and completing the query information based on property graph modeling. Additionally, it provides lower-level APIs for developers who require deeper integration. + +3. **Seamless Integration** + + GOpt is lightweight and serverless, facilitating seamless integration into other databases through a small-sized JAR file deployment. Built on the Calcite framework, GOpt leverages Calcite's extensive range of adapters, simplifying the integration with various data formats. This advantage allows GOpt to seamlessly integrate with mainstream relational databases that has been powered by Calcite. Additionally, GOpt is equipped with graph-native algorithms, enhancing its compatibility with graph-native database APIs. In our [Getting Started](#getting-started) section, we provide a demonstration of GOpt's easy integration into Neo4j with minimal codes. + +## Getting Started + +### Quick Start +GOpt is embeddable and serverless, available as a standalone package occupying just a few tens of MB in a JAR file. It can be quickly integrated into other systems as project dependency via Apache Maven. We have already integrated GOpt into the Neo4j system. Here, you can quickly experience the optimization effects that GOpt brings to Neo4j. + +#### Deployment + +##### Local Setup +We have prepared a `gopt-on-neo4j.tar.gz` package. You can quickly obtain it through the following link: +[Download GOpt for Neo4j](https://graphscope.oss-cn-beijing.aliyuncs.com/gopt_data/gopt-on-neo4j.tar.gz) + +After unzipping the package, you will get the following artifacts: +```bash +β”œβ”€β”€ bin +β”œβ”€β”€ conf +β”œβ”€β”€ data +β”œβ”€β”€ import +β”‚ └── movie +β”œβ”€β”€ lib +β”‚ └── gopt-all-0.0.1-SNAPSHOT.jar +β”œβ”€β”€ plugins +└── queries + β”œβ”€β”€ CBO + β”œβ”€β”€ LSQB + └── LDBC +``` +Based on the original Neo4j artifacts, we made the following modifications: + +1. Added `movie` CSV data to the `import/movie` directory. After successfully importing the data, the corresponding database files will be generated in the `data` directory. +2. Added the `gopt-0.0.1-SNAPSHOT.jar` file to the `lib` directory, making GOpt a project dependency of Neo4j. +3. Added query sets for benchmark tests. + +You can now try running Neo4j with GOpt on your local machine. Note that the Neo4j version we use is 4.4.9. Ensure your local machine has all the necessary environments for [Neo4j-4.4.9](https://github.com/neo4j/neo4j/tree/4.4). If you do not have a complete development environment, you may prefer to use the [Docker setup](#docker-setup). + +##### Docker Setup +If you do not have a complete development environment, we have also prepared a corresponding Docker environment. The Docker image contains all the contents of `gopt-on-neo4j.tar.gz` as well as all the necessary environment dependencies. You can start it with a single command: +```bash +docker run -it registry.cn-hongkong.aliyuncs.com/graphscope/gopt-on-neo4j:latest bash +``` + +#### Importing Data +Use the Neo4j importing tools to load your data into the Neo4j database. Here we use `movie` dataset as an example, which has the following property graph model: + +:::{figure-md} + +movie_schema + +Movie Data Model +::: + +:::{figure-md} + +movie_data + +Nodes and Relationships in the Movie Dataset +::: + +Follow the instructions of [neo4j-admin](https://neo4j.com/docs/operations-manual/4.4/tools/neo4j-admin/) to load the movie dataset, the original CSV files have been included in the `gopt-on-neo4j` artifacts under the directory `import/movie`. +``` +cd gopt-on-neo4j + +bin/neo4j-admin import \ + --database movie \ + --id-type=INTEGER \ + --ignore-empty-strings=true \ + --bad-tolerance=0 \ + --nodes=Person=import/movie/Person.csv \ + --nodes=Movie=import/movie/Movie.csv \ + --relationships=FOLLOWS=import/movie/Person_FOLLOWS_Person.csv \ + --relationships=ACTED_IN=import/movie/Person_ACTED_IN_Movie.csv \ + --delimiter '|' +``` +#### Configuration +We integrate GOpt configuration into the Neo4j standard config file `conf/neo4j.conf` as follows: + +- **graph.planner.rules**: Registers graph relational rules into the GOpt optimizer framework. +- **graph.planner.join.min.pattern.size**: Specifies the minimum pattern size that activates the `JoinDecompositionRule`. It is noted that `Extend` can yield better execution efficiency than `Join` for smaller patterns. +- **graph.planner.intersect.cost.factor**: For performance tuning, a higher value biases the optimizer towards more join-based plans, while a lower value favors extend-based plans. + +Besides the GOpt configuration, we set the default database to `movie`. +``` +# GOpt Configuration +graph.planner.rules=FilterIntoJoinRule,FilterMatchRule,ExtendIntersectRule,JoinDecompositionRule,ExpandGetVFusionRule +graph.planner.join.min.pattern.size:3 +graph.planner.intersect.cost.factor:1 + +# The name of the default database +dbms.default_database=movie +``` + +#### Starting Neo4j Service +```bash +./bin/neo4j start + +# wait unitl neo4j service started +... + +./bin/cypher-shell + +# check graph model +call apoc.meta.graph(); + +# check data +Match (a) Return count(a); +Match (a)-[b]->(c) Return count(b); +``` + +#### Analyzing Queries +To experience the optimization effects of GOpt, you can run various types of queries and observe the detailed profiling results. GOpt integrates Neo4j's Profile tool to display estimated counts for each operation. By comparing these estimates with the actual row counts, you can verify the accuracy of GOpt's cardinality estimation. + +##### Ambiguous Pattern +Execute queries where type constraints are not explicitly defined. +```bash +Profile Match (a)-[]->(b), (b)-[]->(c), (c)-[]->(a) Return a, b, c; +``` +From the execution explanation provided by Neo4j profile tools, it is evident that nodes a, b, and c, as well as the types of relationships between them, are precisely inferred. This allows the GOpt optimizer to operate effectively without being hindered by the lack of type details in queries, addressing Neo4j's limitation where it defaults to rule-based optimization (RBO) in the absence of such information, which prioritizes nodes with defined types. + +:::{figure-md} + +triangle_pattern_profile + +Ambiguous Pattern Profiling Results +::: + +An important point to note is that if the data model does not contain the pattern specified in the query, GOpt can provide timely errors during the compiling phase through static analysis prior to actual data execution. For instance, If GOpt's static analysis determines that there are no outgoing edges associated with `Movie` in the data model, it will promptly raise an error. +```bash +Profile Match (a:Movie)-[]->(b) Return a, b; +``` + +:::{figure-md} + +type_error_profile + +Pattern with Type Error Profiling Results +::: + +##### Graph Pattern Matching +Find all square patterns. +```bash +Profile Match (a)-[]->(b), (a)-[]->(c), (d)-[]->(b), (d)-[]->(c) +Where a<>d AND b<>c +Return a, b, c, d; +``` +Graph-native algorithms (i.e. [worst-case-optimal join](https://dl.acm.org/doi/10.1145/3180143) algorithm) are adopted to optimize the complex graph pattern matching workloads. The WCO implementation is not natively supported in the Neo4j execution layer. However, we can still use GOpt to optimize the order of `Expand` operations in Neo4j. For cyclic patterns, we transform the logical operator generated by the WCO algorithm into Neo4j's specific physical operator, `ExpandInto`. + +:::{figure-md} + +square_pattern_profile + +Square Pattern Profiling Results +::: + +##### Hybrid Graph Relational +Based on the provided filtering conditions, the optimizer seamlessly integrates graph-native algorithm with the traditional relational rule `FilterPushDown`, which results in a distinct execution order from that of query-2. +```bash +Profile Match (a)-[]->(b), (a)-[]->(c), (d)-[]->(b), (d)-[]->(c) +Where d.name = 'Kean' AND a<>d AND b<>c +Return a, b, c, d; +``` + +:::{figure-md} + +square_pattern_with_filter_profile + +Square Pattern with Filtering Profiling Results +::: + +##### ST Path +Find the count of all simple paths between two given vertices. GOpt provides specific optimizations for ST-Path queries. For long paths between a given start and endpoint, it selects an appropriate split point based on cardinality estimation to reduce data expansion in a single direction. This is a feature not available in the native Neo4j planner. +```bash +Profile Match (a)-[:ACTED_IN*5..6]-(b) +Where a.name = 'Kean' AND b.name = 'Top Gun' +Return count(a); +``` + +:::{figure-md} + +st_path_profile + +ST-Path Pattern Profiling Results +::: + +### Installation +We offer two versions of the GOpt JAR package, each approximately 35MB in size: + +- **gopt-core**: Includes the core functionality of GOpt and uses the GraphBuilderSDK to build GOpt inputs. +- **gopt-all**: Builds on top of gopt-core by providing additional Cypher SDK and Gremlin SDK, allowing users to directly write Cypher or Gremlin queries to build GOpt inputs. + +To try GOpt in your own project, download the latest version of the [gopt-core](https://graphscope.oss-cn-beijing.aliyuncs.com/gopt_data/gopt-core-0.0.1-SNAPSHOT.jar) or [gopt-all](https://graphscope.oss-cn-beijing.aliyuncs.com/gopt_data/gopt-all-0.0.1-SNAPSHOT.jar) and place it in the `src/main/resources` directory. +```bash + + gopt + com.alibaba.graphscope + 0.0.1-SNAPSHOT + system + ${project.basedir}/src/main/resources/gopt-core-0.0.1-SNAPSHOT.jar + +``` +```bash + + gopt + com.alibaba.graphscope + 0.0.1-SNAPSHOT + system + ${project.basedir}/src/main/resources/gopt-all-0.0.1-SNAPSHOT.jar + +``` + +## Tutorial + +In the [Getting Started](#getting-started) section, we explored some basic features of GOpt. In this section, we will guide you through advanced features of GOpt and demonstrate how to leverage these capabilities to optimize your system, including: +- **GOpt Benchmark Tools**: This guide provides detailed instructions on utilizing the benchmark tool to evaluate GOpt's performance on more complex queries with larger datasets. +- **Integration of GOpt into Other Systems**: Using Neo4j as a case study, we provide comprehensive details on integrating GOpt into other systems at different levels. + +### Benchmark Tools +We provide a benchmark tool for running complex pattern queries on larger datasets. Our benchmark tool includes three components: + +- **Dataset**: We have pre-prepared the LDBC data with sf=1 in the artifacts `gopt-on-neo4j`, under `data` directory. +- **Queries**: We have prepared three sets of query collections in the `queries` directory, including: + 1. **CBO**: Designed for commonly used graph patterns, including triangles, squares, paths, and 4-cliques. + 2. **LSQB**: A subset of standard LSQB queries. + 3. **LDBC**: Simplified versions of standard LDBC queries. +- **Scripts**: The `bin/bench.sh` script is used to run the benchmark. + +You can now experience our one-click benchmarking feature. + +#### Steps + +1. First, stop the running service + ```bash + bin/neo4j stop + ``` + +2. Reconfigure the database to `ldbc1`, and set the performance tuning configuration for GOpt + ``` + dbms.default_database=ldbc1 + graph.planner.intersect.cost.factor:3 + ``` + +3. Restart the Neo4j service, and confirm that the data is ready + ```bash + bin/neo4j start + # wait unitl neo4j service started + ... + + ./bin/cypher-shell + + # check graph model + call apoc.meta.graph(); + + # check data + Match (a) Return count(a); + ``` + +4. Experience the various features of `bench.sh` by passing different parameters. + 1. Run the CBO query set in profile mode to print detailed profiling results. Alternatively, you can use explain mode, which returns the optimized plan without executing it. + ```bash + ./bin/bench.sh ./queries/CBO --profile + ./bin/bench.sh ./queries/CBO --explain + ``` + + 2. You can choose other query sets or run all query sets under `queries` + ```bash + ./bin/bench.sh ./queries/LSQB --profile + ./bin/bench.sh ./queries/LDBC --profile + ./bin/bench.sh ./queries --profile + ``` + + 3. By default, we skip large queries that are time-consuming. You can use `--all` to run all queries, but this will take more time due to Neo4j's low execution efficiency on single node resources. + ```bash + ./bin/bench.sh ./queries --profile --all + ``` + + 4. Optionally, you can switch to the Neo4j native planner by setting `--neo4j`. This allows you to analyze the differences from GOpt. Be sure to clear the query caches before switching between planners. + ```bash + # clean query cache + ./bin/cypher-shell 'call db.clearQueryCaches();' + + ./bin/bench.sh ./queries --profile --neo4j --all + ``` + +### Integration of GOpt +Detailed integration instructions for GOpt will be provided in the following sections. Stay tuned for more information. + +## Design of GOpt +### Motivation +In large-scale graph querying scenarios, one core technique that can significantly influence performance is the optimizer. As graph data and its applications continue to grow rapidly, numerous graph processing systems equipped with native optimizers have been developed. However, these optimization methods tend to excessively focus on either graph-specific or relational-specific approaches. This narrow focus fails to holistically address queries involving both graph and relational data, leading to suboptimal performance. +To address this issue, GOpt is designed as follows: + +1. **Unified Graph Relational Design**: Develop a unified structure for Graph Relational that can express Gremlin or Cypher queries, serving as the initial structure for optimization. +2. **Unified Optimization Framework**: Optimize the graph relational structure within a unified relational optimization framework. We extended the interfaces of traditional relational frameworks to implement advanced graph optimizations, including high-order statistics and worst-case optimal join algorithms. +3. **Graph-Specific Optimization**: We also employed specific optimizations for graph pattern matching based on the characteristics of graphs. For instance, we extended type inference for graph patterns based on property graph modeling, leveraged graph isomorphism for pruning in traditional optimization frameworks, and implemented a specialized graph decomposition algorithm to reduce the search space during the optimization process. + +:::{figure-md} + +design_of_gopt + +Design of GOpt +::: + +### Detailed Introduction +A comprehensive introduction to GOpt will be provided in subsequent sections. Please stay tuned for detailed updates and information. diff --git a/docs/interactive_engine/optimizer.md b/docs/interactive_engine/optimizer.md deleted file mode 100644 index 6a3e45482a2b..000000000000 --- a/docs/interactive_engine/optimizer.md +++ /dev/null @@ -1,140 +0,0 @@ -# A Generic Graph Optimizer for GIE - -We have developed a versatile graph optimizer for the Graph Interactive Engine (GIE), capable of optimizing the execution for multiple query languages in GraphScope, including [Gremlin](./tinkerpop_eco) and [Cypher](./neo4j_eco). This optimizer is built upon the [Apache Calcite](https://calcite.apache.org/) framework, traditionally focused on relational databases, but now extended to accommodate graph databases by incorporating graph-specific operations and rules. The optimizer consists of two main components: the Rule-based Optimizer (RBO) and the Cost-based Optimizer (CBO). Like their relational counterparts, both RBO and CBO utilize pre-registered rules for transformation and path optimization, either through heuristic methods or cost estimation. However, our optimizer differs from standard relational optimizers in two key aspects: - -1. **Graph-specific Rules**: Our optimizer integrates rules for specific graph operators within Graph and Relational databases, allowing for true optimization where Graph and Relational structures coexist. Relational operators directly benefit from the reuse of existing SQL rules. - -2. **High-Order Statistics in CBO**: Our Cost-Based Optimizer (CBO) utilizes sophisticated statistical models grounded in graph structures, providing more accurate cardinality estimates compared to conventional optimizers. The foundational work for our CBO primarily stems from the research presented in [GlogS](https://www.usenix.org/conference/atc23/presentation/lai), which was published at ATC 2023. This research has been instrumental in enhancing the efficacy of graph pattern matching, setting a new standard for optimization in graph databases. - -## Rule-based Optimizer (RBO) - -| Rule Name | Description | -|------------------------|--------------------------------------------------------------| -| FilterMatchRule | Pushes filter conditions to graph operators in `Match`. | -| DegreeFusionRule | Uses pre-calculated degrees of vertices from graph storage. | -| NotMatchToAntiJoinRule | Converts `Where Not` queries into anti-joins. | -| FieldTrimRule | Removes unnecessary aliases or properties. | - -This table offers a concise overview of the existing rules for RBO. Each rule is designed to optimize graph query performance by transforming the way queries are processed and executed. Examples of how these rules conceptually affect the execution of queries are detailed as follows. Note that we use Cypher as the query language for demonstration purposes, but the rules apply to Gremlin as well. - -### FilterMatchRule -This rule pushes filter conditions down to individual graph operators within a `Match` clause. For example: -```bash -Match (p1:PERSON)-[:KNOWS]->(p2:PERSON) -Where p1.id = $id1 and p2.id = $id2 -Return p1, p2; -``` -becomes: -```bash -Match (p1:PERSON {id: $id1})-[:KNOWS]->(p2:PERSON {id: $id2}) -Return p1, p2; -``` -following the push-down, splitting the `Where` clause to filter directly at the vertices p1 and p2. - -### DegreeFusionRule -The DegreeFusionRule capitalizes on the fact that most graph storage systems already maintain the count of neighbors (i.e., the degree) for each vertex. For example: -```bash -Match (p1:PERSON {id: $id1})-[]->(p2) -Return p1, COUNT(p2); -``` -With this rule applied, instead of individually counting each neighbor of `p1`, we can directly obtain `p1`'s degree, streamlining the process considerably. This optimization effectively utilizes preexisting graph data structures to expedite query execution. - -### NotMatchToAntiJoinRule -This rule converts queries containing `Where Not` clauses into the equivalent anti-join structures, where the anti-join -is a relational operator that returns all rows from the left table that fail to match with the right table. -Consider the following query: -```bash -Match (p1:PERSON)-[:KNOWS]->(:PERSON)-[:KNOWS]->(p2:PERSON) -Where Not (p1)-[:KNOWS]->(p2) -Return p1, p2; -``` -Under this rule, the query is transformed to: -```bash -Match (p1:PERSON)-[:KNOWS]->(:PERSON)-[:KNOWS]->(p2:PERSON) - -Match (p1)-[:KNOWS]->(p2) -Return p1, p2; -``` -Here, `` serves as a conceptual operator, used here to effectively illustrate how the original query's intent is preserved while enhancing its execution efficiency. - -### FieldTrimRule -Removes unnecessary aliases or properties, reducing data processing. Consider: -```bash -Match (p1:PERSON)-[k1:KNOWS]->(p2:PERSON)-[k2:KNOWS]->(p3:PERSON) -Return p1.name; -``` -Here, only `p1.name` is required in the output, making aliases like k1, k2, and properties of p2, p3 redundant. -This rule optimizes to retain only necessary data, significantly reducing the volume of the intermediate results. - -## Cost-based Optimizer (CBO) -TODO - -## Graph Type Inference -Traditional SQL only provides checks and inference for basic data types, such as int, double, string, boolean, etc. However, graph databases encompass more complex types. Beyond basic data types, they introduce special vertex and edge types. Vertex types constrain the range of entity types within the graph, while edge types constrain relationships between entities. While the introduction of vertex and edge types in graph databases provides a more flexible data model, it simultaneously adds complexity to the data, necessitating type checks and inference for graph data. - -We address this complexity through Graph Type Inference, offering functionality for type checks and inference for graph data. This feature examines whether vertex and edge types in graph data adhere to user-defined schemas and can infer vertex and edge types, laying the groundwork for subsequent optimizations in graph queries. - -Taking the example of a modern graph schema, which includes the following vertex and edge types: -``` -# Vertex types -person (name, age) -software (name, lang) - -# Edge types -knows (weight) -created (weight) -``` -Graph Type Inference provides the following capabilities: -- Checking if vertex and edge types in the graph conform to user-defined schemas: - - Reporting errors for nonexistent vertex or edge types: - ```bash - # 'per' is a nonexistent vertex type - Match (a:per) Return a; - ``` - ```bash - # 'kno' is a nonexistent edge type - Match (a:person)-[b:kno]->(c) Return a, b, c; - ``` - - Reporting errors for nonexistent properties or mismatched property types in vertices or edges: - ```bash - # The 'lang' property does not exist in vertices of type 'person' - Match (a:person {lang: 'java'}) Return a; - ``` - ```bash - # The 'name' property in vertices of type 'person' is of type string, cannot perform addition - Match (a:person) Return a.name + 1; - ``` - - Reporting errors for invalid paths composed of vertices and edges: - ```bash - # There is no edge of type 'created' between vertices of type 'person' - Match (a:person)-[:created]->(b:person) - ``` - ```bash - # Vertices of type 'software' have no outgoing edges - Match (a:software)-[b]->(c); - ``` -- Inferring vertex and edge types in the graph: - - Inferring vertex types given edge types: - ```bash - # (?)-[knows]->(?) => (person)-[knows]->(person) - Match (a)-[:knows]->(b) Return labels(a), labels(b); - ``` - ```bash - # (?)-[knows*1..4]->(?) => (person)-[knows*1..2]->(person) - Match (a)-[:knows*1..4]->(b) Return labels(a), labels(b); - ``` - - Inferring edge types given vertex types: - ```bash - # (person)-[?]->(person) => (person)-[knows]->(person) - Match (a:person)-[b]->(c:person) Return type(b); - ``` - - Inferring all vertex and edge types given a path: - ```bash - # (?)-[?]->(?)->[?]->(software) => (person)-[knows]->(person)->[created]->(software) - Match (a)-[b]->(c)-[d]->(:software) Return labels(a), type(b), labels(c), type(d); - ``` - - Inferring across multiple sentences: - ```bash - # (?)-[]->(?), (?)-[knows]-(?) => (person)-[knows]-(person), (person)-[knows]-(person) - Match (a)-[b]-(c), (a)-[:KNOWS]-(c) Return labels(a), type(b), labels(c); - ``` diff --git a/flex/bin/bulk_loader.cc b/flex/bin/bulk_loader.cc index 6157640df3c9..c7a55679d4fd 100644 --- a/flex/bin/bulk_loader.cc +++ b/flex/bin/bulk_loader.cc @@ -164,8 +164,14 @@ int main(int argc, char** argv) { auto loader = gs::LoaderFactory::CreateFragmentLoader( data_dir_path.string(), schema_res.value(), loading_config_res.value()); - loader->LoadFragment(); + auto result = loader->LoadFragment(); + if (!result.ok()) { + std::filesystem::remove_all(data_dir_path); + LOG(ERROR) << "Failed to load fragment: " + << result.status().error_message(); + return -1; + } t += grape::GetCurrentTime(); LOG(INFO) << "Finished bulk loading in " << t << " seconds."; diff --git a/flex/bin/load_plan_and_gen.sh b/flex/bin/load_plan_and_gen.sh index 6a62b077c034..6fae6a1e5200 100755 --- a/flex/bin/load_plan_and_gen.sh +++ b/flex/bin/load_plan_and_gen.sh @@ -345,6 +345,12 @@ compile_hqps_so() { info "Finish building, output to ${output_so_path}" popd + # strip the output_so_path + strip ${output_so_path} + # clean the cmake directory, the cmake files may take up a lot of space + cmd="rm -rf ${cur_dir}/CMakeFiles" + eval ${cmd} + ################### now copy ########################## # if dst_so_path eq output_so_path, skip copying. if [ ${dst_so_path} == ${output_so_path} ]; then diff --git a/flex/engines/graph_db/grin/CMakeLists.txt b/flex/engines/graph_db/grin/CMakeLists.txt index de9f6af9cf2a..9198340d8a35 100644 --- a/flex/engines/graph_db/grin/CMakeLists.txt +++ b/flex/engines/graph_db/grin/CMakeLists.txt @@ -69,9 +69,11 @@ add_custom_target(grin_clformat COMMAND clang-format --style=file -i ${FILES_NEED_FORMAT} COMMENT "Running clang-format." VERBATIM) +file(GLOB_RECURSE SRC_SOURCES "src/*.cc") +file(GLOB_RECURSE UTILS_SOURCES "../../../utils/*.cc") +file(GLOB_RECURSE STORAGE_SOURCES "../../../storages/rt_mutable_graph/*.cc") +set(SOURCES ${SRC_SOURCES} ${UTILS_SOURCES} ${STORAGE_SOURCES}) -file(GLOB SOURCES "src/*.cc" "src/topology/*.cc" "src/property/*.cc" "src/index/*.cc" "src/common/*.cc" "../../../utils/property/*.cc" "../../../utils/*.cc" - "../../../storages/rt_mutable_graph/*.cc" "../../../storages/rt_mutable_graph/loader/*.cc") add_library(flex_grin SHARED ${SOURCES}) target_link_libraries(flex_grin ${LIBGRAPELITE_LIBRARIES} ${GFLAGS_LIBRARIES} ${CMAKE_DL_LIBS} ${YAML_CPP_LIBRARIES}) if (ARROW_SHARED_LIB) diff --git a/flex/engines/graph_db/grin/src/property/property.cc b/flex/engines/graph_db/grin/src/property/property.cc index 74c1b19fa455..aa008fad20bd 100644 --- a/flex/engines/graph_db/grin/src/property/property.cc +++ b/flex/engines/graph_db/grin/src/property/property.cc @@ -54,7 +54,7 @@ GRIN_VERTEX_PROPERTY_LIST grin_get_vertex_properties_by_name(GRIN_GRAPH g, std::string prop_name(name); auto vps = new GRIN_VERTEX_PROPERTY_LIST_T(); std::string _name = std::string(name); - for (auto idx = 0; idx < _g->g.vertex_label_num_; idx++) { + for (size_t idx = 0; idx < _g->g.vertex_label_num_; idx++) { auto& table = _g->g.get_vertex_table(static_cast(idx)); auto col = table.get_column(name); diff --git a/flex/engines/graph_db/grin/src/property/propertylist.cc b/flex/engines/graph_db/grin/src/property/propertylist.cc index a52639f105c9..1d2b6485a1f9 100644 --- a/flex/engines/graph_db/grin/src/property/propertylist.cc +++ b/flex/engines/graph_db/grin/src/property/propertylist.cc @@ -122,7 +122,7 @@ GRIN_EDGE_PROPERTY grin_get_edge_property_from_list(GRIN_GRAPH g, size_t idx) { auto _epl = static_cast(epl); if (_epl->size() <= idx) { - return GRIN_NULL_EDGE_PROPERTY; + return static_cast(GRIN_NULL_EDGE_PROPERTY); } return (*_epl)[idx]; } diff --git a/flex/engines/graph_db/runtime/common/rt_any.cc b/flex/engines/graph_db/runtime/common/rt_any.cc index c611a0925451..84383154ed4b 100644 --- a/flex/engines/graph_db/runtime/common/rt_any.cc +++ b/flex/engines/graph_db/runtime/common/rt_any.cc @@ -578,9 +578,21 @@ void RTAny::sink(const gs::ReadTransaction& txn, int id, e->set_id(encode_unique_edge_id(edge_label, src, dst)); auto& prop_names = txn.schema().get_edge_property_names( label.src_label, label.dst_label, label.edge_label); - auto props = e->add_properties(); - props->mutable_key()->set_name(prop_names[0]); - sink_any(prop, e->mutable_properties(0)->mutable_value()); + if (prop_names.size() == 1) { + auto props = e->add_properties(); + props->mutable_key()->set_name(prop_names[0]); + sink_any(prop, e->mutable_properties(0)->mutable_value()); + } else if (prop_names.size() > 1) { + auto rv = prop.AsRecordView(); + if (rv.size() != prop_names.size()) { + LOG(ERROR) << "record view size not match with prop names"; + } + for (size_t i = 0; i < prop_names.size(); ++i) { + auto props = e->add_properties(); + props->mutable_key()->set_name(prop_names[i]); + sink_any(rv[i], props->mutable_value()); + } + } } else if (type_ == RTAnyType::kPath) { LOG(FATAL) << "not support path sink"; diff --git a/flex/engines/hqps_db/core/operator/path_expand.h b/flex/engines/hqps_db/core/operator/path_expand.h index 28e1acce79c6..2ce52927f21d 100644 --- a/flex/engines/hqps_db/core/operator/path_expand.h +++ b/flex/engines/hqps_db/core/operator/path_expand.h @@ -131,7 +131,6 @@ class PathExpand { for (auto j = range.start_; j < range.limit_; ++j) { next_path.clear(); auto& cur_offset_vec = other_offsets[j]; - CHECK(cur_path.size() == vertices_vec.size()); next_path.resize(vertices_vec.size()); for (size_t i = 0; i < vertices_vec.size(); ++i) { diff --git a/flex/engines/hqps_db/core/operator/sort.h b/flex/engines/hqps_db/core/operator/sort.h index 7abc89921c6b..aa4d50838e1c 100644 --- a/flex/engines/hqps_db/core/operator/sort.h +++ b/flex/engines/hqps_db/core/operator/sort.h @@ -412,7 +412,9 @@ class SortOp { static auto create_prop_getter_impl_for_order_pair( const ORDER_PAIR& ordering_pair, const Collection& set, const GRAPH_INTERFACE& graph) { - CHECK(ordering_pair.name == "None" || ordering_pair.name == "none"); + if (ordering_pair.name != "None" || ordering_pair.name != "none") { + throw std::runtime_error("Expect None property getter for Collection."); + } return CollectionPropGetter(); } }; diff --git a/flex/engines/hqps_db/database/mutable_csr_interface.h b/flex/engines/hqps_db/database/mutable_csr_interface.h index febd6202b430..b98d55ba2d2b 100644 --- a/flex/engines/hqps_db/database/mutable_csr_interface.h +++ b/flex/engines/hqps_db/database/mutable_csr_interface.h @@ -21,6 +21,7 @@ #include "flex/engines/graph_db/database/graph_db_session.h" #include "flex/engines/hqps_db/core/null_record.h" #include "flex/engines/hqps_db/core/params.h" +#include "flex/utils/exception.h" #include "flex/engines/hqps_db/database/adj_list.h" #include "grape/utils/bitset.h" @@ -294,8 +295,8 @@ class MutableCSRInterface { const label_id_t& label_id, const std::vector& vids, const std::array>>& prop_names) const { - // auto label_id = db_session_.schema().get_vertex_label_id(label); - CHECK(label_id < db_session_.schema().vertex_label_num()); + THROW_EXCEPTION_IF(label_id >= db_session_.schema().vertex_label_num(), + "Invalid label id: " + std::to_string(label_id)); std::tuple>...> columns; get_tuple_column_from_graph(label_id, prop_names, columns); std::vector> props(vids.size()); @@ -388,7 +389,8 @@ class MutableCSRInterface { std::vector> props(total_size); std::vector label_ids; for (label_id_t label : labels) { - CHECK(label < db_session_.schema().vertex_label_num()); + THROW_EXCEPTION_IF(label >= db_session_.schema().vertex_label_num(), + "Invalid label id: " + std::to_string(label)); label_ids.emplace_back(label); } using column_tuple_t = std::tuple>...>; @@ -409,8 +411,6 @@ class MutableCSRInterface { std::vector& columns, const std::vector& vids, const grape::Bitset& bitset) const { - // auto index_seq = std::make_index_sequence{}; - { auto& column_tuple0 = columns[0]; auto& column_tuple1 = columns[1]; @@ -571,11 +571,8 @@ class MutableCSRInterface { edge_label_id); auto csr1 = db_session_.graph().get_ie_csr(dst_label_id, src_label_id, edge_label_id); - // CHECK(csr0); - // CHECK(csr1); return mutable_csr_graph_impl::AdjListArray(csr0, csr1, vids); } else { - // LOG(FATAL) << "Not implemented - " << direction_str; throw std::runtime_error("Not implemented - " + direction_str); } } diff --git a/flex/engines/http_server/actor/admin_actor.act.cc b/flex/engines/http_server/actor/admin_actor.act.cc index f93d84ff1664..6cdb2de1b0ae 100644 --- a/flex/engines/http_server/actor/admin_actor.act.cc +++ b/flex/engines/http_server/actor/admin_actor.act.cc @@ -96,7 +96,12 @@ std::string merge_graph_and_plugin_meta( nlohmann::json res; for (auto& graph_meta : res_graph_metas) { - res.push_back(nlohmann::json::parse(graph_meta.ToJson())); + try { + res.push_back(nlohmann::json::parse(graph_meta.ToJson())); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to parse graph meta: " << e.what() + << graph_meta.ToJson(); + } } return res.empty() ? "{}" : res.dump(); } @@ -348,20 +353,38 @@ gs::Status invoke_delete_plugin_meta( // util functions -std::string to_json_str(const std::vector& plugin_metas) { - nlohmann::json res; - for (auto& plugin_meta : plugin_metas) { - res.push_back(nlohmann::json::parse(plugin_meta.ToJson())); +gs::Result to_json_str( + const std::vector& plugin_metas) { + try { + nlohmann::json res; + for (auto& plugin_meta : plugin_metas) { + res.push_back(nlohmann::json::parse(plugin_meta.ToJson())); + } + return res.empty() ? gs::Result("{}") + : gs::Result(res.dump()); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to parse plugin meta from json string: " << e.what(); + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to parse plugin meta: " + std::string(e.what()))); } - return res.empty() ? "{}" : res.dump(); } -std::string to_json_str(const std::vector& job_metas) { - nlohmann::json res; - for (auto& job_meta : job_metas) { - res.push_back(nlohmann::json::parse(job_meta.ToJson(true))); +gs::Result to_json_str( + const std::vector& job_metas) { + try { + nlohmann::json res; + for (auto& job_meta : job_metas) { + res.push_back(nlohmann::json::parse(job_meta.ToJson())); + } + return res.empty() ? gs::Result("{}") + : gs::Result(res.dump()); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to parse job meta from json string: " << e.what(); + return gs::Result( + gs::Status(gs::StatusCode::InternalError, + "Fail to parse job meta: " + std::string(e.what()))); } - return res.empty() ? "{}" : res.dump(); } admin_actor::~admin_actor() { @@ -711,7 +734,7 @@ seastar::future admin_actor::get_procedures_by_graph_name( graph_meta_res.value().plugin_metas.begin(), graph_meta_res.value().plugin_metas.end()); return seastar::make_ready_future( - gs::Result(to_json_str(all_plugin_metas))); + to_json_str(all_plugin_metas)); } else { LOG(ERROR) << "Fail to get all procedures: " << get_all_procedure_res.status().error_message(); @@ -1118,6 +1141,7 @@ seastar::future admin_actor::service_status( auto running_graph_res = metadata_store_->GetRunningGraph(); nlohmann::json res; if (query_port != 0) { + res["statistics_enabled"] = true; // default is true res["status"] = graph_db_service.is_actors_running() ? "Running" : "Stopped"; res["hqps_port"] = query_port; @@ -1145,7 +1169,15 @@ seastar::future admin_actor::service_status( graph_meta.plugin_metas.emplace_back(plugin_meta); } } - res["graph"] = nlohmann::json::parse(graph_meta.ToJson()); + try { + res["graph"] = nlohmann::json::parse(graph_meta.ToJson()); + } catch (std::exception& e) { + LOG(ERROR) << "Fail to parse graph meta: " << e.what(); + return seastar::make_exception_future( + gs::Status( + gs::StatusCode::InternalError, + "Fail to parse graph meta: " + std::string(e.what()))); + } } else { LOG(ERROR) << "Fail to get all procedures: " << get_all_procedure_res.status().error_message(); @@ -1221,9 +1253,8 @@ seastar::future admin_actor::list_jobs( auto list_res = metadata_store_->GetAllJobMeta(); if (list_res.ok()) { VLOG(10) << "Successfully list jobs"; - auto list_job_metas_str = to_json_str(list_res.value()); return seastar::make_ready_future( - gs::Result(std::move(list_job_metas_str))); + to_json_str(list_res.value())); } else { LOG(ERROR) << "Fail to list jobs: " << list_res.status().error_message(); return seastar::make_ready_future(list_res.status()); diff --git a/flex/openapi/openapi_interactive.yaml b/flex/openapi/openapi_interactive.yaml index 24f995149b4a..49ec336df076 100644 --- a/flex/openapi/openapi_interactive.yaml +++ b/flex/openapi/openapi_interactive.yaml @@ -1791,6 +1791,8 @@ components: x-body-name: service_status type: object properties: + statistics_enabled: # indicate whether the graph statistics interface is enabled + type: boolean status: type: string graph: diff --git a/flex/resources/hqps/CMakeLists.txt.template b/flex/resources/hqps/CMakeLists.txt.template index c1d2ce5660f3..563736b77a6d 100644 --- a/flex/resources/hqps/CMakeLists.txt.template +++ b/flex/resources/hqps/CMakeLists.txt.template @@ -38,8 +38,12 @@ if(EXISTS "/opt/graphscope/include") include_directories("/opt/graphscope/include") endif() +if (EXISTS "/opt/flex/include") + include_directories("/opt/flex/include") +endif() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O0 -flto -Werror=unused-result -fPIC -no-pie") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fopenmp -Wl,-rpath,$ORIGIN -O2 -flto -Werror=unused-result -fPIC -no-pie") add_library(${QUERY_NAME} SHARED ${PROJECT_SOURCE_DIR}/${QUERY_NAME}.cc) target_include_directories(${QUERY_NAME} PUBLIC ${FLEX_INCLUDE_PREFIX} ${FLEX_INCLUDE_PREFIX}/flex/build/engines/hqps_db/) diff --git a/flex/storages/metadata/default_graph_meta_store.cc b/flex/storages/metadata/default_graph_meta_store.cc index b2e2397945e5..73a8c6bf886e 100644 --- a/flex/storages/metadata/default_graph_meta_store.cc +++ b/flex/storages/metadata/default_graph_meta_store.cc @@ -81,8 +81,9 @@ Result DefaultGraphMetaStore::UpdateGraphMeta( json = nlohmann::json::parse(old_meta); } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old graph meta:" << e.what(); - return Result(Status(StatusCode::InternalError, - "Fail to parse old graph meta")); + return Result( + Status(StatusCode::InternalError, + std::string("Fail to parse old graph meta: ") + e.what())); } auto graph_meta = GraphMeta::FromJson(json); if (request.graph_name.has_value()) { @@ -197,8 +198,9 @@ Result DefaultGraphMetaStore::UpdatePluginMeta( json = nlohmann::json::parse(old_meta); } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old plugin meta:" << e.what(); - return Result(Status(StatusCode::InternalError, - "Fail to parse old plugin meta")); + return Result(Status( + StatusCode::InternalError, + std::string("Fail to parse old plugin meta: ") + e.what())); } auto plugin_meta = PluginMeta::FromJson(json); if (plugin_meta.bound_graph != graph_id) { @@ -288,7 +290,8 @@ Result DefaultGraphMetaStore::UpdateJobMeta( } catch (const std::exception& e) { LOG(ERROR) << "Fail to parse old job meta:" << e.what(); return Result( - Status(StatusCode::InternalError, "Fail to parse old job meta")); + Status(StatusCode::InternalError, + std::string("Fail to parse old job meta: ") + e.what())); } auto job_meta = JobMeta::FromJson(json); if (update_request.status.has_value()) { diff --git a/flex/storages/metadata/graph_meta_store.cc b/flex/storages/metadata/graph_meta_store.cc index 44801276ab91..74b1e25ec92a 100644 --- a/flex/storages/metadata/graph_meta_store.cc +++ b/flex/storages/metadata/graph_meta_store.cc @@ -65,21 +65,40 @@ std::string GraphMeta::ToJson() const { json["creation_time"] = creation_time; json["data_update_time"] = data_update_time; if (!data_import_config.empty()) { - json["data_import_config"] = nlohmann::json::parse(data_import_config); + try { + json["data_import_config"] = nlohmann::json::parse(data_import_config); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid data_import_config: " << data_import_config << " " + << e.what(); + } + } + try { + json["schema"] = nlohmann::json::parse(schema); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid schema: " << schema << " " << e.what(); } - json["schema"] = nlohmann::json::parse(schema); json["stored_procedures"] = nlohmann::json::array(); for (auto& plugin_meta : plugin_metas) { - json["stored_procedures"].push_back( - nlohmann::json::parse(plugin_meta.ToJson())); + try { + json["stored_procedures"].push_back( + nlohmann::json::parse(plugin_meta.ToJson())); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid plugin_meta: " << plugin_meta.ToJson() << " " + << e.what(); + } } json["store_type"] = store_type; return json.dump(); } GraphMeta GraphMeta::FromJson(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - return GraphMeta::FromJson(j); + try { + auto j = nlohmann::json::parse(json_str); + return GraphMeta::FromJson(j); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid json string: " << json_str << " " << e.what(); + return GraphMeta(); + } } GraphMeta GraphMeta::FromJson(const nlohmann::json& json) { @@ -119,8 +138,13 @@ GraphMeta GraphMeta::FromJson(const nlohmann::json& json) { } PluginMeta PluginMeta::FromJson(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - return PluginMeta::FromJson(j); + try { + auto j = nlohmann::json::parse(json_str); + return PluginMeta::FromJson(j); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid json string: " << json_str << " " << e.what(); + return PluginMeta(); + } } PluginMeta PluginMeta::FromJson(const nlohmann::json& json) { @@ -217,33 +241,46 @@ void PluginMeta::setParamsFromJsonString(const std::string& json_str) { json_str == "nu") { return; } - auto j = nlohmann::json::parse(json_str); - if (j.is_array()) { - for (auto& param : j) { - Parameter p; - p.name = param["name"].get(); - p.type = param["type"].get(); - params.push_back(p); + try { + auto j = nlohmann::json::parse(json_str); + if (j.is_array()) { + for (auto& param : j) { + Parameter p; + p.name = param["name"].get(); + p.type = param["type"].get(); + params.push_back(p); + } + } else { + LOG(ERROR) << "Invalid params string: " << json_str; } - } else { - LOG(ERROR) << "Invalid params string: " << json_str; + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid params string: " << json_str << " " << e.what(); } } void PluginMeta::setReturnsFromJsonString(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - for (auto& ret : j) { - Parameter p; - p.name = ret["name"].get(); - p.type = ret["type"].get(); - returns.push_back(p); + try { + auto j = nlohmann::json::parse(json_str); + for (auto& ret : j) { + Parameter p; + p.name = ret["name"].get(); + p.type = ret["type"].get(); + returns.push_back(p); + } + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid returns string: " << json_str << " " << e.what(); } } void PluginMeta::setOptionFromJsonString(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - for (auto& opt : j.items()) { - option[opt.key()] = opt.value().get(); + try { + auto j = nlohmann::json::parse(json_str); + for (auto& opt : j.items()) { + option[opt.key()] = opt.value().get(); + } + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid option string: " << json_str; + return; } } @@ -265,8 +302,14 @@ std::string JobMeta::ToJson(bool print_log) const { } JobMeta JobMeta::FromJson(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - return JobMeta::FromJson(j); + try { + auto j = nlohmann::json::parse(json_str); + return JobMeta::FromJson(j); + } catch (const std::exception& e) { + LOG(ERROR) << "Fail to parse JobMeta from json: " << json_str << " " + << e.what(); + return JobMeta(); + } } JobMeta JobMeta::FromJson(const nlohmann::json& json) { @@ -354,7 +397,11 @@ std::string CreateGraphMetaRequest::ToString() const { nlohmann::json json; json["name"] = name; json["description"] = description; - json["schema"] = nlohmann::json::parse(schema); + try { + json["schema"] = nlohmann::json::parse(schema); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid schema: " << schema << " " << e.what(); + } if (data_update_time.has_value()) { json["data_update_time"] = data_update_time.value(); } else { @@ -363,8 +410,13 @@ std::string CreateGraphMetaRequest::ToString() const { json["creation_time"] = creation_time; json["stored_procedures"] = nlohmann::json::array(); for (auto& plugin_meta : plugin_metas) { - json["stored_procedures"].push_back( - nlohmann::json::parse(plugin_meta.ToJson())); + try { + json["stored_procedures"].push_back( + nlohmann::json::parse(plugin_meta.ToJson())); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid plugin_meta: " << plugin_meta.ToJson() << " " + << e.what(); + } } return json.dump(); } @@ -438,8 +490,14 @@ std::string CreatePluginMetaRequest::ToString() const { CreatePluginMetaRequest CreatePluginMetaRequest::FromJson( const std::string& json) { - auto j = nlohmann::json::parse(json); - return CreatePluginMetaRequest::FromJson(j); + try { + auto j = nlohmann::json::parse(json); + return CreatePluginMetaRequest::FromJson(j); + } catch (const std::exception& e) { + LOG(ERROR) << "CreatePluginMetaRequest::FromJson error: " << json << ", " + << e.what(); + return CreatePluginMetaRequest(); + } } CreatePluginMetaRequest CreatePluginMetaRequest::FromJson( @@ -561,7 +619,8 @@ UpdatePluginMetaRequest UpdatePluginMetaRequest::FromJson( request.enable = j["enable"].get(); } } catch (const std::exception& e) { - LOG(ERROR) << "UpdatePluginMetaRequest::FromJson error: " << e.what(); + LOG(ERROR) << "UpdatePluginMetaRequest::FromJson error: " << e.what() << " " + << json; } return request; } @@ -741,8 +800,16 @@ std::string GraphStatistics::ToJson() const { } Result GraphStatistics::FromJson(const std::string& json_str) { - auto j = nlohmann::json::parse(json_str); - return GraphStatistics::FromJson(j); + try { + auto j = nlohmann::json::parse(json_str); + return GraphStatistics::FromJson(j); + } catch (const std::exception& e) { + LOG(ERROR) << "Invalid json string: " << json_str << " " << e.what(); + return Result(Status( + StatusCode::InternalError, + "Invalid json string when parsing graph statistics : " + json_str + + " " + e.what())); + } } Result GraphStatistics::FromJson(const nlohmann::json& json) { diff --git a/flex/storages/rt_mutable_graph/csr/mutable_csr.cc b/flex/storages/rt_mutable_graph/csr/mutable_csr.cc index 3949cd80f789..0751dad5a381 100644 --- a/flex/storages/rt_mutable_graph/csr/mutable_csr.cc +++ b/flex/storages/rt_mutable_graph/csr/mutable_csr.cc @@ -20,6 +20,58 @@ namespace gs { +void read_file(const std::string& filename, void* buffer, size_t size, + size_t num) { + FILE* fin = fopen(filename.c_str(), "r"); + if (fin == nullptr) { + std::stringstream ss; + ss << "Failed to open file " << filename << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + size_t ret_len = 0; + if ((ret_len = fread(buffer, size, num, fin)) != num) { + std::stringstream ss; + ss << "Failed to read file " << filename << ", expected " << num << ", got " + << ret_len << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + int ret = 0; + if ((ret = fclose(fin)) != 0) { + std::stringstream ss; + ss << "Failed to close file " << filename << ", error code: " << ret << " " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } +} + +void write_file(const std::string& filename, const void* buffer, size_t size, + size_t num) { + FILE* fout = fopen(filename.c_str(), "wb"); + if (fout == nullptr) { + std::stringstream ss; + ss << "Failed to open file " << filename << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + size_t ret_len = 0; + if ((ret_len = fwrite(buffer, size, num, fout)) != num) { + std::stringstream ss; + ss << "Failed to write file " << filename << ", expected " << num + << ", got " << ret_len << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + } + int ret = 0; + if ((ret = fclose(fout)) != 0) { + std::stringstream ss; + ss << "Failed to close file " << filename << ", error code: " << ret << " " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } +} template class SingleMutableCsr; template class MutableCsr; diff --git a/flex/storages/rt_mutable_graph/csr/mutable_csr.h b/flex/storages/rt_mutable_graph/csr/mutable_csr.h index 5996122b2ea8..fdf52cd45e0a 100644 --- a/flex/storages/rt_mutable_graph/csr/mutable_csr.h +++ b/flex/storages/rt_mutable_graph/csr/mutable_csr.h @@ -26,6 +26,12 @@ namespace gs { +void read_file(const std::string& filename, void* buffer, size_t size, + size_t num); + +void write_file(const std::string& filename, const void* buffer, size_t size, + size_t num); + template class MutableCsrConstEdgeIter : public CsrConstEdgeIterBase { using const_nbr_ptr_t = typename MutableNbrSlice::const_nbr_ptr_t; @@ -391,28 +397,63 @@ class MutableCsr : public TypedMutableCsrBase { } if (need_cap_list) { - FILE* fcap_out = - fopen((new_snapshot_dir + "/" + name + ".cap").c_str(), "wb"); - CHECK_EQ(fwrite(cap_list.data(), sizeof(int), cap_list.size(), fcap_out), - cap_list.size()); - fflush(fcap_out); - fclose(fcap_out); + write_file(new_snapshot_dir + "/" + name + ".cap", cap_list.data(), + sizeof(int), cap_list.size()); } if (reuse_nbr_list && !nbr_list_.filename().empty() && std::filesystem::exists(nbr_list_.filename())) { + std::error_code errorCode; std::filesystem::create_hard_link(nbr_list_.filename(), - new_snapshot_dir + "/" + name + ".nbr"); + new_snapshot_dir + "/" + name + ".nbr", + errorCode); + if (errorCode) { + std::stringstream ss; + ss << "Failed to create hard link from " << nbr_list_.filename() + << " to " << new_snapshot_dir + "/" + name + ".snbr" + << ", error code: " << errorCode << " " << errorCode.message(); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } } else { FILE* fout = fopen((new_snapshot_dir + "/" + name + ".nbr").c_str(), "wb"); + std::string filename = new_snapshot_dir + "/" + name + ".nbr"; + if (fout == nullptr) { + std::stringstream ss; + ss << "Failed to open nbr list " << filename << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + for (size_t i = 0; i < vnum; ++i) { - CHECK_EQ(fwrite(adj_lists_[i].data(), sizeof(nbr_t), - adj_lists_[i].capacity(), fout), - adj_lists_[i].capacity()); + size_t ret{}; + if ((ret = fwrite(adj_lists_[i].data(), sizeof(nbr_t), + adj_lists_[i].capacity(), fout)) != + static_cast(adj_lists_[i].capacity())) { + std::stringstream ss; + ss << "Failed to write nbr list " << filename << ", expected " + << adj_lists_[i].capacity() << ", got " << ret << ", " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + } + int ret = 0; + if ((ret = fflush(fout)) != 0) { + std::stringstream ss; + ss << "Failed to flush nbr list " << filename << ", error code: " << ret + << " " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } + if ((ret = fclose(fout)) != 0) { + std::stringstream ss; + ss << "Failed to close nbr list " << filename << ", error code: " << ret + << " " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } - fflush(fout); - fclose(fout); } } @@ -510,10 +551,8 @@ class MutableCsr : public TypedMutableCsrBase { void load_meta(const std::string& prefix) { std::string meta_file_path = prefix + ".meta"; if (std::filesystem::exists(meta_file_path)) { - FILE* meta_file_fd = fopen(meta_file_path.c_str(), "r"); - CHECK_EQ(fread(&unsorted_since_, sizeof(timestamp_t), 1, meta_file_fd), - 1); - fclose(meta_file_fd); + read_file(meta_file_path, &unsorted_since_, sizeof(timestamp_t), 1); + } else { unsorted_since_ = 0; } @@ -521,10 +560,7 @@ class MutableCsr : public TypedMutableCsrBase { void dump_meta(const std::string& prefix) const { std::string meta_file_path = prefix + ".meta"; - FILE* meta_file_fd = fopen((prefix + ".meta").c_str(), "wb"); - CHECK_EQ(fwrite(&unsorted_since_, sizeof(timestamp_t), 1, meta_file_fd), 1); - fflush(meta_file_fd); - fclose(meta_file_fd); + write_file(meta_file_path, &unsorted_since_, sizeof(timestamp_t), 1); } grape::SpinLock* locks_; @@ -768,9 +804,7 @@ class SingleMutableCsr : public TypedMutableCsrBase { size_t old_size = nbr_list_.size(); nbr_list_.reset(); nbr_list_.resize(v_cap); - FILE* fin = fopen((prefix + ".snbr").c_str(), "r"); - CHECK_EQ(fread(nbr_list_.data(), sizeof(nbr_t), old_size, fin), old_size); - fclose(fin); + read_file(prefix + ".snbr", nbr_list_.data(), sizeof(nbr_t), old_size); for (size_t k = old_size; k != v_cap; ++k) { nbr_list_[k].timestamp.store(std::numeric_limits::max()); } @@ -792,13 +826,21 @@ class SingleMutableCsr : public TypedMutableCsrBase { const std::string& new_snapshot_dir) override { if ((!nbr_list_.filename().empty() && std::filesystem::exists(nbr_list_.filename()))) { - std::filesystem::create_hard_link( - nbr_list_.filename(), new_snapshot_dir + "/" + name + ".snbr"); + std::error_code errorCode; + std::filesystem::create_hard_link(nbr_list_.filename(), + new_snapshot_dir + "/" + name + ".snbr", + errorCode); + if (errorCode) { + std::stringstream ss; + ss << "Failed to create hard link from " << nbr_list_.filename() + << " to " << new_snapshot_dir + "/" + name + ".snbr" + << ", error code: " << errorCode << " " << errorCode.message(); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } } else { - FILE* fp = fopen((new_snapshot_dir + "/" + name + ".snbr").c_str(), "wb"); - fwrite(nbr_list_.data(), sizeof(nbr_t), nbr_list_.size(), fp); - fflush(fp); - fclose(fp); + write_file(new_snapshot_dir + "/" + name + ".snbr", nbr_list_.data(), + sizeof(nbr_t), nbr_list_.size()); } } diff --git a/flex/storages/rt_mutable_graph/file_names.h b/flex/storages/rt_mutable_graph/file_names.h index 7b01b1fbe613..64e7d25c758a 100644 --- a/flex/storages/rt_mutable_graph/file_names.h +++ b/flex/storages/rt_mutable_graph/file_names.h @@ -166,6 +166,7 @@ inline std::string get_latest_snapshot(const std::string& work_dir) { { FILE* fin = fopen((snapshots_dir + "/VERSION").c_str(), "r"); CHECK_EQ(fread(&version, sizeof(uint32_t), 1, fin), 1); + fclose(fin); } return snapshots_dir + "/" + std::to_string(version); } diff --git a/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.cc index b1c817c9640c..6024b3b4086d 100644 --- a/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.cc @@ -12,12 +12,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +#include -#include "flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h" #include "flex/engines/hqps_db/core/utils/hqps_utils.h" +#include "flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h" namespace gs { +void printDiskRemaining(const std::string& path) { + struct statvfs buf; + if (statvfs(path.c_str(), &buf) == 0) { + LOG(INFO) << "Disk remaining: " << buf.f_bsize * buf.f_bavail / 1024 / 1024 + << "MB"; + } +} + bool check_primary_key_type(std::shared_ptr data_type) { if (data_type->Equals(arrow::int64()) || data_type->Equals(arrow::uint64()) || data_type->Equals(arrow::int32()) || data_type->Equals(arrow::uint32()) || diff --git a/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h b/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h index fe297884a21f..b8097b34536a 100644 --- a/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h +++ b/flex/storages/rt_mutable_graph/loader/abstract_arrow_fragment_loader.h @@ -33,7 +33,9 @@ namespace gs { +void printDiskRemaining(const std::string& path); // The interface providing visitor pattern for RecordBatch. + class IRecordBatchSupplier { public: virtual ~IRecordBatchSupplier() = default; diff --git a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc index 40bee19a14a7..87795b15e334 100644 --- a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.cc @@ -728,11 +728,21 @@ void CSVFragmentLoader::loadEdges() { } } -void CSVFragmentLoader::LoadFragment() { - loadVertices(); - loadEdges(); - - basic_fragment_loader_.LoadFragment(); +Result CSVFragmentLoader::LoadFragment() { + try { + loadVertices(); + loadEdges(); + + basic_fragment_loader_.LoadFragment(); + } catch (const std::exception& e) { + auto work_dir = basic_fragment_loader_.work_dir(); + printDiskRemaining(work_dir); + LOG(ERROR) << "Load fragment failed: " << e.what(); + return Result(StatusCode::InternalError, + "Load fragment failed: " + std::string(e.what()), + false); + } + return Result(true); } const bool CSVFragmentLoader::registered_ = LoaderFactory::Register( diff --git a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.h b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.h index cea7555caa87..3cf8e7d3972b 100644 --- a/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.h +++ b/flex/storages/rt_mutable_graph/loader/csv_fragment_loader.h @@ -75,7 +75,7 @@ class CSVFragmentLoader : public AbstractArrowFragmentLoader { ~CSVFragmentLoader() {} - void LoadFragment() override; + Result LoadFragment() override; private: void loadVertices(); diff --git a/flex/storages/rt_mutable_graph/loader/i_fragment_loader.h b/flex/storages/rt_mutable_graph/loader/i_fragment_loader.h index 8bf4b6de4476..4bf4d646e630 100644 --- a/flex/storages/rt_mutable_graph/loader/i_fragment_loader.h +++ b/flex/storages/rt_mutable_graph/loader/i_fragment_loader.h @@ -24,7 +24,7 @@ namespace gs { class IFragmentLoader { public: virtual ~IFragmentLoader() = default; - virtual void LoadFragment() = 0; + virtual Result LoadFragment() = 0; }; } // namespace gs diff --git a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc index afc05833996d..b90dc11b5b40 100644 --- a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc +++ b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.cc @@ -347,12 +347,22 @@ std::shared_ptr ODPSFragmentLoader::Make( } void ODPSFragmentLoader::init() { odps_read_client_.init(); } -void ODPSFragmentLoader::LoadFragment() { - init(); - loadVertices(); - loadEdges(); - - basic_fragment_loader_.LoadFragment(); +Result ODPSFragmentLoader::LoadFragment() { + try { + init(); + loadVertices(); + loadEdges(); + + basic_fragment_loader_.LoadFragment(); + } catch (const std::exception& e) { + auto work_dir = basic_fragment_loader_.work_dir(); + printDiskRemaining(work_dir); + LOG(ERROR) << "Failed to load fragment: " << e.what(); + return Result(StatusCode::InternalError, + "Load fragment failed: " + std::string(e.what()), + false); + } + return Result(true); } // odps_table_path is like /project_name/table_name/partition_name diff --git a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.h b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.h index 1da86c5a27d0..026bb954f63d 100644 --- a/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.h +++ b/flex/storages/rt_mutable_graph/loader/odps_fragment_loader.h @@ -180,7 +180,7 @@ class ODPSFragmentLoader : public AbstractArrowFragmentLoader { ~ODPSFragmentLoader() {} - void LoadFragment() override; + Result LoadFragment() override; private: void init(); diff --git a/flex/storages/rt_mutable_graph/loading_config.cc b/flex/storages/rt_mutable_graph/loading_config.cc index d17cf47e3200..4b3ba345d786 100644 --- a/flex/storages/rt_mutable_graph/loading_config.cc +++ b/flex/storages/rt_mutable_graph/loading_config.cc @@ -21,6 +21,7 @@ #include #include #include "flex/engines/hqps_db/core/utils/hqps_utils.h" +#include "flex/utils/exception.h" namespace gs { @@ -892,8 +893,9 @@ LoadingConfig::GetEdgeLoadingMeta() const { const std::vector>& LoadingConfig::GetVertexColumnMappings(label_t label_id) const { - CHECK(vertex_column_mappings_.find(label_id) != - vertex_column_mappings_.end()); + THROW_EXCEPTION_IF( + vertex_column_mappings_.find(label_id) == vertex_column_mappings_.end(), + "Vertex label id not found in vertex column mappings"); return vertex_column_mappings_.at(label_id); } @@ -901,7 +903,9 @@ const std::vector>& LoadingConfig::GetEdgeColumnMappings(label_t src_label_id, label_t dst_label_id, label_t edge_label_id) const { auto key = std::make_tuple(src_label_id, dst_label_id, edge_label_id); - CHECK(edge_column_mappings_.find(key) != edge_column_mappings_.end()); + THROW_EXCEPTION_IF( + edge_column_mappings_.find(key) == edge_column_mappings_.end(), + "Edge label id not found in edge column mappings"); return edge_column_mappings_.at(key); } @@ -910,7 +914,8 @@ const std::pair>, LoadingConfig::GetEdgeSrcDstCol(label_t src_label_id, label_t dst_label_id, label_t edge_label_id) const { auto key = std::make_tuple(src_label_id, dst_label_id, edge_label_id); - CHECK(edge_src_dst_col_.find(key) != edge_src_dst_col_.end()); + THROW_EXCEPTION_IF(edge_src_dst_col_.find(key) == edge_src_dst_col_.end(), + "Edge label id not found in edge column mappings"); return edge_src_dst_col_.at(key); } diff --git a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc index bf3daacf69e4..4fc437820046 100644 --- a/flex/storages/rt_mutable_graph/mutable_property_fragment.cc +++ b/flex/storages/rt_mutable_graph/mutable_property_fragment.cc @@ -299,7 +299,15 @@ void MutablePropertyFragment::Compact(uint32_t version) { void MutablePropertyFragment::Dump(const std::string& work_dir, uint32_t version) { std::string snapshot_dir_path = snapshot_dir(work_dir, version); - std::filesystem::create_directories(snapshot_dir_path); + std::error_code errorCode; + std::filesystem::create_directories(snapshot_dir_path, errorCode); + if (errorCode) { + std::stringstream ss; + ss << "Failed to create snapshot directory: " << snapshot_dir_path << ", " + << errorCode.message(); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); + } std::vector vertex_num(vertex_label_num_, 0); for (size_t i = 0; i < vertex_label_num_; ++i) { vertex_num[i] = lf_indexers_[i].size(); diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index 35c88020167f..65fd95dd655b 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -14,17 +14,13 @@ */ #include "flex/storages/rt_mutable_graph/schema.h" +#include "flex/utils/exception.h" #include namespace gs { -#define THROW_EXCEPTION_IF(cond, msg) \ - if (cond) { \ - throw std::runtime_error(msg); \ - } - -Schema::Schema() : has_multi_props_edge_(false) {}; +Schema::Schema() : has_multi_props_edge_(false){}; Schema::~Schema() = default; void Schema::Clear() { diff --git a/flex/tests/hqps/hqps_adhoc_test.sh b/flex/tests/hqps/hqps_adhoc_test.sh index a5f9968fddfb..019499bb9da2 100644 --- a/flex/tests/hqps/hqps_adhoc_test.sh +++ b/flex/tests/hqps/hqps_adhoc_test.sh @@ -188,6 +188,7 @@ run_cypher_test(){ else echo "GRAPH_NAME: ${GRAPH_NAME} not supported, use movies, ldbc or graph_algo" fi + rm -rf /tmp/neo4j-* || true } diff --git a/flex/utils/exception.h b/flex/utils/exception.h new file mode 100644 index 000000000000..c71563c6740f --- /dev/null +++ b/flex/utils/exception.h @@ -0,0 +1,23 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef UTILS_EXCEPTION_H_ +#define UTILS_EXCEPTION_H_ + +#define THROW_EXCEPTION_IF(cond, msg) \ + if (cond) { \ + throw std::runtime_error(msg); \ + } + +#endif // UTILS_EXCEPTION_H_ \ No newline at end of file diff --git a/flex/utils/mmap_array.h b/flex/utils/mmap_array.h index 3cb5d74128d0..4e821ece40e7 100644 --- a/flex/utils/mmap_array.h +++ b/flex/utils/mmap_array.h @@ -79,13 +79,16 @@ class mmap_array { } mmap_array(mmap_array&& rhs) : mmap_array() { swap(rhs); } - ~mmap_array() {} + ~mmap_array() { reset(); } void reset() { if (data_ != NULL && mmap_size_ != 0) { if (munmap(data_, mmap_size_) != 0) { - LOG(FATAL) << "Failed to mummap file [ " << filename_ << " ] " - << strerror(errno); + std::stringstream ss; + ss << "Failed to mummap file [ " << filename_ << " ] " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } data_ = NULL; @@ -93,8 +96,11 @@ class mmap_array { mmap_size_ = 0; if (fd_ != -1) { if (close(fd_) != 0) { - LOG(FATAL) << "Failed to close file [ " << filename_ << " ] " - << strerror(errno); + std::stringstream ss; + ss << "Failed to close file [ " << filename_ << " ] " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } fd_ = -1; } @@ -107,8 +113,11 @@ class mmap_array { reset(); if (old_filename != "" && std::filesystem::exists(old_filename)) { if (std::filesystem::remove(old_filename) == 0) { - LOG(FATAL) << "Failed to remove file [ " << old_filename << " ] " - << strerror(errno); + std::stringstream ss; + ss << "Failed to remove file [ " << old_filename << " ] " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } } @@ -126,8 +135,10 @@ class mmap_array { bool creat = !std::filesystem::exists(filename_); fd_ = ::open(filename_.c_str(), O_RDWR | O_CREAT, 0777); if (fd_ == -1) { - LOG(FATAL) << "Failed to open file [" << filename_ << "], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to open file [" << filename_ << "], " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (creat) { std::filesystem::perms readWritePermission = @@ -138,8 +149,11 @@ class mmap_array { std::filesystem::perm_options::add, errorCode); if (errorCode) { - LOG(FATAL) << "Failed to set read/write permission for file: " - << filename << " " << errorCode.message() << std::endl; + std::stringstream ss; + ss << "Failed to set read/write permission for file: " << filename + << " " << errorCode.message() << std::endl; + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } @@ -152,13 +166,19 @@ class mmap_array { data_ = reinterpret_cast( mmap(NULL, mmap_size_, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0)); if (data_ == MAP_FAILED) { - LOG(FATAL) << "Failed to mmap file [" << filename_ << "], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to mmap file [" << filename_ << "], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } int rt = madvise(data_, mmap_size_, MADV_RANDOM | MADV_WILLNEED); if (rt != 0) { - LOG(FATAL) << "Failed to madvise file [" << filename_ << "], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to madvise file [" << filename_ << "], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } } else { @@ -166,8 +186,11 @@ class mmap_array { size_t file_size = std::filesystem::file_size(filename_); fd_ = ::open(filename_.c_str(), O_RDWR, 0777); if (fd_ == -1) { - LOG(FATAL) << "Failed to open file [" << filename_ << "], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to open file [" << filename_ << "], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } size_ = file_size / sizeof(T); mmap_size_ = file_size; @@ -177,8 +200,11 @@ class mmap_array { data_ = reinterpret_cast(mmap( NULL, mmap_size_, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd_, 0)); if (data_ == MAP_FAILED) { - LOG(FATAL) << "Failed to mmap file [" << filename_ << "], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to mmap file [" << filename_ << "], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } } @@ -198,16 +224,25 @@ class mmap_array { if (data_ != MAP_FAILED) { FILE* fin = fopen(filename.c_str(), "rb"); if (fin == NULL) { - LOG(FATAL) << "Failed to open file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to open file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (fread(data_, sizeof(T), size_, fin) != size_) { - LOG(FATAL) << "Failed to fread file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to fread file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (fclose(fin) != 0) { - LOG(FATAL) << "Failed to fclose file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to fclose file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } else { LOG(ERROR) << "allocating hugepage failed, " << strerror(errno) @@ -228,26 +263,40 @@ class mmap_array { std::error_code errorCode; std::filesystem::rename(old_filename, filename, errorCode); if (errorCode) { - LOG(FATAL) << "Failed to rename file " << old_filename << " to " - << filename << " " << errorCode.message() << std::endl; + std::stringstream ss; + ss << "Failed to rename file " << old_filename << " to " << filename + << " " << errorCode.message() << std::endl; + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } else { FILE* fout = fopen(filename.c_str(), "wb"); if (fout == NULL) { - LOG(FATAL) << "Failed to open file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to open file [ " << filename << " ], " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (fwrite(data_, sizeof(T), size_, fout) != size_) { - LOG(FATAL) << "Failed to fwrite file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to fwrite file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (fflush(fout) != 0) { - LOG(FATAL) << "Failed to fflush file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to fflush file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (fclose(fout) != 0) { - LOG(FATAL) << "Failed to fclose file [ " << filename << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to fclose file [ " << filename << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } reset(); } @@ -259,8 +308,11 @@ class mmap_array { std::filesystem::perm_options::add, errorCode); if (errorCode) { - LOG(FATAL) << "Failed to set read permission for file: " << filename - << " " << errorCode.message() << std::endl; + std::stringstream ss; + ss << "Failed to set read permission for file: " << filename << " " + << errorCode.message() << std::endl; + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } @@ -272,14 +324,20 @@ class mmap_array { if (sync_to_file_) { if (data_ != NULL && mmap_size_ != 0) { if (munmap(data_, mmap_size_) != 0) { - LOG(FATAL) << "Failed to mummap file [ " << filename_ << " ], " - << strerror(errno); + std::stringstream ss; + ss << "Failed to mummap file [ " << filename_ << " ], " + << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } size_t new_mmap_size = size * sizeof(T); int rt = ftruncate(fd_, new_mmap_size); if (rt == -1) { - LOG(FATAL) << "Failed to ftruncate " << rt << ", " << strerror(errno); + std::stringstream ss; + ss << "Failed to ftruncate " << rt << ", " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } if (new_mmap_size == 0) { data_ = NULL; @@ -287,7 +345,10 @@ class mmap_array { data_ = reinterpret_cast(mmap( NULL, new_mmap_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd_, 0)); if (data_ == MAP_FAILED) { - LOG(FATAL) << "Failed to mmap, " << strerror(errno); + std::stringstream ss; + ss << "Failed to mmap, " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } size_ = size; @@ -314,7 +375,10 @@ class mmap_array { mmap(NULL, new_mmap_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)); if (new_data == MAP_FAILED) { - LOG(FATAL) << "mmap failed " << strerror(errno); + std::stringstream ss; + ss << "mmap failed " << strerror(errno); + LOG(ERROR) << ss.str(); + throw std::runtime_error(ss.str()); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/StoreConfig.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/StoreConfig.java index 35ba45a24570..db83389a63f1 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/StoreConfig.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/StoreConfig.java @@ -24,7 +24,7 @@ public class StoreConfig { Config.intConfig("store.write.thread.count", 1); public static final Config STORE_QUEUE_BUFFER_SIZE = - Config.intConfig("store.queue.buffer.size", 102400); + Config.intConfig("store.queue.buffer.size", 1024000); public static final Config STORE_QUEUE_WAIT_MS = Config.longConfig("store.queue.wait.ms", 3000L); diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/SchemaFetcher.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/SchemaFetcher.java index df74a612cdde..14f9cc366e01 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/SchemaFetcher.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/SchemaFetcher.java @@ -25,4 +25,6 @@ public interface SchemaFetcher { int getPartitionNum(); int getVersion(); + + boolean statisticsEnabled(); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java index eec6443d43e6..ed7b5260ed07 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/fetcher/DynamicIrMetaFetcher.java @@ -20,6 +20,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.GraphConfig; +import com.alibaba.graphscope.common.ir.meta.GraphId; import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.IrMetaStats; import com.alibaba.graphscope.common.ir.meta.IrMetaTracker; @@ -44,18 +45,19 @@ public class DynamicIrMetaFetcher extends IrMetaFetcher implements AutoCloseable private volatile IrMetaStats currentState; // To manage the state changes of statistics resulting from different update operations. private volatile StatsState statsState; + private volatile Boolean statsEnabled = null; public DynamicIrMetaFetcher(Configs configs, IrMetaReader dataReader, IrMetaTracker tracker) { super(dataReader, tracker); this.scheduler = new ScheduledThreadPoolExecutor(2); this.scheduler.scheduleAtFixedRate( () -> syncMeta(), - 0, + 2000, GraphConfig.GRAPH_META_SCHEMA_FETCH_INTERVAL_MS.get(configs), TimeUnit.MILLISECONDS); this.scheduler.scheduleAtFixedRate( - () -> syncStats(), - 0, + () -> syncStats(statsEnabled == null ? false : statsEnabled), + 2000, GraphConfig.GRAPH_META_STATISTICS_FETCH_INTERVAL_MS.get(configs), TimeUnit.MILLISECONDS); } @@ -68,10 +70,14 @@ public Optional fetch() { private synchronized void syncMeta() { try { IrMeta meta = this.reader.readMeta(); + logger.debug( + "schema from remote: {}", + (meta == null) ? null : meta.getSchema().schemaJson()); GraphStatistics curStats; // if the graph id is changed, we need to update the statistics if (this.currentState == null - || !this.currentState.getGraphId().equals(meta.getGraphId())) { + || !this.currentState.getGraphId().equals(meta.getGraphId()) + || this.currentState.getSnapshotId().getId() != meta.getSnapshotId().getId()) { this.statsState = StatsState.INITIALIZED; curStats = null; } else { @@ -84,19 +90,35 @@ private synchronized void syncMeta() { meta.getSchema(), meta.getStoredProcedures(), curStats); - if (this.statsState != StatsState.SYNCED) { - syncStats(); + boolean statsEnabled = getStatsEnabled(this.currentState.getGraphId()); + if (statsEnabled && this.statsState != StatsState.SYNCED + || (!statsEnabled && this.statsState != StatsState.MOCKED)) { + logger.debug("start to sync stats"); + syncStats(statsEnabled); } - } catch (Exception e) { + } catch (Throwable e) { logger.warn("failed to read meta data, error is {}", e); } } - private synchronized void syncStats() { + private boolean getStatsEnabled(GraphId graphId) { try { - if (this.currentState != null) { + return this.statsEnabled == null + ? this.reader.syncStatsEnabled(graphId) + : this.statsEnabled; + } catch ( + Throwable e) { // if errors happen when reading stats enabled, we assume it is false + logger.warn("failed to read stats enabled, error is {}", e); + return false; + } + } + + private synchronized void syncStats(boolean statsEnabled) { + try { + if (this.currentState != null && statsEnabled) { GraphStatistics stats = this.reader.readStats(this.currentState.getGraphId()); - if (stats != null) { + logger.debug("statistics from remote: {}", stats); + if (stats != null && stats.getVertexCount() != 0) { this.currentState = new IrMetaStats( this.currentState.getSnapshotId(), @@ -104,19 +126,25 @@ private synchronized void syncStats() { this.currentState.getStoredProcedures(), stats); if (tracker != null) { + logger.debug("start to update the glogue"); tracker.onChanged(this.currentState); } this.statsState = StatsState.SYNCED; } } - } catch (Exception e) { + } catch (Throwable e) { logger.warn("failed to read graph statistics, error is {}", e); } finally { - if (this.currentState != null - && tracker != null - && this.statsState == StatsState.INITIALIZED) { - tracker.onChanged(this.currentState); - this.statsState = StatsState.MOCKED; + try { + if (this.currentState != null + && tracker != null + && this.statsState == StatsState.INITIALIZED) { + logger.debug("start to mock the glogue"); + tracker.onChanged(this.currentState); + this.statsState = StatsState.MOCKED; + } + } catch (Throwable t) { + logger.warn("failed to mock the glogue, error is {}", t); } } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java index 05ae98478633..c67949c56124 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/HttpIrMetaReader.java @@ -108,6 +108,23 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { } } + @Override + public boolean syncStatsEnabled(GraphId graphId) throws IOException { + try { + HttpResponse response = + sendRequest(GraphConfig.GRAPH_META_SCHEMA_URI.get(configs)); + String res = response.body(); + Preconditions.checkArgument( + response.statusCode() == 200, + "read service status fail, status code: %s, error message: %s", + response.statusCode(), + res); + return getStaticEnabled(res); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + private HttpResponse sendRequest(String requestUri) throws IOException, InterruptedException { HttpRequest request = @@ -128,4 +145,14 @@ private Pair convertMetaFromJsonToYaml(String metaInJson) throw Yaml yaml = new Yaml(); return Pair.with(graphId, yaml.dump(metaMap)); } + + private boolean getStaticEnabled(String metaInJson) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(metaInJson); + Map rootMap = mapper.convertValue(rootNode, Map.class); + if (rootMap.containsKey("statistics_enabled")) { + return (boolean) rootMap.get("statistics_enabled"); + } + return false; // default value + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java index 62e4142684b3..a6a808bf2dc4 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/IrMetaReader.java @@ -30,4 +30,7 @@ public interface IrMetaReader { // get statistics from a graph referenced by graphId GraphStatistics readStats(GraphId graphId) throws IOException; + + // a synchronous invocation to check whether statistics functionality is enabled in the backend + boolean syncStatsEnabled(GraphId graphId) throws IOException; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java index 7c0db6727816..8c8c96458036 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/reader/LocalIrMetaReader.java @@ -78,4 +78,10 @@ public IrGraphStatistics readStats(GraphId graphId) throws IOException { (statsURI.getScheme() == null) ? Path.of(statsURI.getPath()) : Path.of(statsURI); return new IrGraphStatistics(new FileInputStream(statsPath.toFile())); } + + @Override + public boolean syncStatsEnabled(GraphId graphId) { + String statsUri = GraphConfig.GRAPH_META_STATISTICS_URI.get(configs); + return !statsUri.isEmpty(); + } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java index ee24b7aa3a16..702d0a1fcdfd 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/glogue/GlogueBasicCardinalityEstimationImpl.java @@ -161,7 +161,8 @@ public Double getCardinality(Pattern queryPattern) { return this.patternCardinality.get(pattern); } } - return 0.0; + // if not exist, return 1.0 + return 1.0; } @Override diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java index 3ec2d1c3af7e..7333418806ab 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/rel/metadata/schema/GlogueSchema.java @@ -21,6 +21,8 @@ import org.jgrapht.Graph; import org.jgrapht.graph.DirectedPseudograph; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; @@ -30,6 +32,7 @@ public class GlogueSchema { private Graph schemaGraph; private HashMap vertexTypeCardinality; private HashMap edgeTypeCardinality; + private static Logger logger = LoggerFactory.getLogger(GlogueSchema.class); public GlogueSchema( GraphSchema graphSchema, @@ -69,6 +72,7 @@ public GlogueSchema(GraphSchema graphSchema) { edgeTypeCardinality.put(edgeType, 1.0); } } + logger.debug("GlogueSchema created with default cardinality 1.0: {}", this); } public GlogueSchema(GraphSchema graphSchema, GraphStatistics statistics) { @@ -108,6 +112,7 @@ public GlogueSchema(GraphSchema graphSchema, GraphStatistics statistics) { } } } + logger.debug("GlogueSchema created with statistics: {}", this); } public static GlogueSchema fromMeta(IrMetaStats irMeta) { @@ -139,7 +144,9 @@ public List getEdgeTypes(Integer source, Integer target) { public Double getVertexTypeCardinality(Integer vertexType) { Double cardinality = this.vertexTypeCardinality.get(vertexType); if (cardinality == null) { - return 0.0; + logger.debug( + "Vertex type {} not found in schema, assuming cardinality 1.0", vertexType); + return 1.0; } else { return cardinality; } @@ -148,7 +155,8 @@ public Double getVertexTypeCardinality(Integer vertexType) { public Double getEdgeTypeCardinality(EdgeTypeId edgeType) { Double cardinality = this.edgeTypeCardinality.get(edgeType); if (cardinality == null) { - return 0.0; + logger.debug("Edge type {} not found in schema, assuming cardinality 1.0", edgeType); + return 1.0; } else { return cardinality; } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java index c2835eee2274..69184c613934 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/GraphRelToProtoConverter.java @@ -121,7 +121,7 @@ public RelNode visit(GraphLogicalSource source) { queryParamsBuilder, Utils.extractColumnsFromRelDataType(source.getRowType(), isColumnId)); } - scanBuilder.setParams(buildQueryParams(source)); + scanBuilder.setParams(queryParamsBuilder); if (source.getAliasId() != AliasInference.DEFAULT_ID) { scanBuilder.setAlias(Utils.asAliasId(source.getAliasId())); } @@ -396,6 +396,8 @@ public RelNode visit(LogicalFilter filter) { Map> tagColumns = Utils.extractTagColumnsFromRexNodes(List.of(filter.getCondition())); if (preCacheEdgeProps) { + // Currently, we've already precache edge properties and path properties, so we + // need to remove them. So as the follows. Utils.removeEdgeProperties( com.alibaba.graphscope.common.ir.tools.Utils.getOutputType( filter.getInput()), diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java index 9ed2b40b1f36..86a354cd2e0f 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/runtime/proto/Utils.java @@ -22,6 +22,7 @@ import com.alibaba.graphscope.common.ir.tools.config.GraphOpt; import com.alibaba.graphscope.common.ir.type.GraphLabelType; import com.alibaba.graphscope.common.ir.type.GraphNameOrId; +import com.alibaba.graphscope.common.ir.type.GraphPathType; import com.alibaba.graphscope.common.ir.type.GraphProperty; import com.alibaba.graphscope.common.ir.type.GraphSchemaType; import com.alibaba.graphscope.gaia.proto.*; @@ -738,7 +739,7 @@ public static Set extractColumnsFromRelDataType( return columns; } - // remove edge properties from columns by checking if the tags refers to edge type + // remove properties from columns by checking if the tags refers to edge type or path type public static void removeEdgeProperties( RelDataType inputDataType, Map> tagColumns) { List fieldTypes = inputDataType.getFieldList(); @@ -750,19 +751,25 @@ public static void removeEdgeProperties( && GraphOpt.Source.EDGE.equals( ((GraphSchemaType) headFieldType.getType()).getScanOpt())) { tags.remove(AliasInference.DEFAULT_ID); + } else if (headFieldType.getType() instanceof GraphPathType) { + tags.remove(AliasInference.DEFAULT_ID); } } + if (tags.isEmpty()) { return; } - // then, process other tags by checking if they are of edge type + // then, process other tags by checking if they are of edge type or path type List removeKeys = new ArrayList<>(); for (RelDataTypeField fieldType : fieldTypes) { - if (tags.contains(fieldType.getIndex()) - && fieldType.getType() instanceof GraphSchemaType - && GraphOpt.Source.EDGE.equals( - ((GraphSchemaType) fieldType.getType()).getScanOpt())) { - removeKeys.add(fieldType.getIndex()); + if (tags.contains(fieldType.getIndex())) { + if (fieldType.getType() instanceof GraphSchemaType + && GraphOpt.Source.EDGE.equals( + ((GraphSchemaType) fieldType.getType()).getScanOpt())) { + removeKeys.add(fieldType.getIndex()); + } else if (fieldType.getType() instanceof GraphPathType) { + removeKeys.add(fieldType.getIndex()); + } } } tagColumns.keySet().removeAll(removeKeys); diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeInference.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeInference.java index 5efd51bc12c7..de516330b390 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeInference.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/type/GraphTypeInference.java @@ -32,6 +32,7 @@ import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.rel.RelNode; import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; import org.apache.calcite.rex.RexLiteral; import org.apache.calcite.rex.RexNode; import org.apache.commons.lang3.ObjectUtils; @@ -700,7 +701,10 @@ private GraphSchemaType createSchemaType( boolean isNullable = originalType == null ? false : originalType.isNullable(); if (newLabels.size() == 1) { return new GraphSchemaType( - opt, new GraphLabelType(newLabels), ImmutableList.of(), isNullable); + opt, + new GraphLabelType(newLabels), + getOriginalFields(newLabels.get(0), originalType), + isNullable); } else { List fuzzyTypes = newLabels.stream() @@ -709,12 +713,24 @@ private GraphSchemaType createSchemaType( new GraphSchemaType( opt, new GraphLabelType(ImmutableList.of(k)), - ImmutableList.of())) + getOriginalFields(k, originalType))) .collect(Collectors.toList()); return GraphSchemaType.create(fuzzyTypes, builder.getTypeFactory(), isNullable); } } + private List getOriginalFields( + GraphLabelType.Entry labelEntry, @Nullable GraphSchemaType originalType) { + if (originalType == null) return ImmutableList.of(); + List candidates = originalType.getSchemaTypeAsList(); + for (GraphSchemaType candidate : candidates) { + if (candidate.getLabelType().getLabelsEntry().contains(labelEntry)) { + return candidate.getFieldList(); + } + } + return ImmutableList.of(); + } + private class RelGraph { private final Map> aliasNameToRels; private final List rels; diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GLogueSchemaTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GLogueSchemaTest.java index 86b27ee9459c..c0f0fa84d34c 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GLogueSchemaTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GLogueSchemaTest.java @@ -43,10 +43,12 @@ public void glogue_schema_test() { // test get type cardinality Assert.assertEquals(4.0, g.getVertexTypeCardinality(person), delta); Assert.assertEquals(2.0, g.getVertexTypeCardinality(software), delta); - Assert.assertEquals(0.0, g.getVertexTypeCardinality(2), delta); Assert.assertEquals(2.0, g.getEdgeTypeCardinality(knows), delta); Assert.assertEquals(4.0, g.getEdgeTypeCardinality(creates), delta); - Assert.assertEquals(0.0, g.getEdgeTypeCardinality(new EdgeTypeId(0, 0, 2)), delta); + + // when query a type that is not in the schema, return 1.0 + Assert.assertEquals(1.0, g.getVertexTypeCardinality(2), delta); + Assert.assertEquals(1.0, g.getEdgeTypeCardinality(new EdgeTypeId(0, 0, 2)), delta); // test get types Assert.assertEquals(2, g.getAdjEdgeTypes(person).size()); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GlogueTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GlogueTest.java index 9a589b354152..2949c44e40ec 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GlogueTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/common/ir/rel/metadata/GlogueTest.java @@ -77,6 +77,7 @@ public void glogue_get_row_count_test() { Assert.assertEquals(1.0d, count2, delta); // pattern3: person2 <- person1 <- software (not exist) + // Note the non-exist pattern will return a cardinality of 1.0 Pattern pattern3 = new Pattern(); pattern3.addVertex(v0); pattern3.addVertex(v1); @@ -86,7 +87,7 @@ public void glogue_get_row_count_test() { pattern3.reordering(); Double count3 = gl.getRowCount(pattern3); - Assert.assertEquals(0.0d, count3, delta); + Assert.assertEquals(1.0d, count3, delta); } @Test diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java index c19d71ab85ea..9fac91413579 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/antlr4/MatchTest.java @@ -19,6 +19,7 @@ import com.alibaba.graphscope.common.config.Configs; import com.alibaba.graphscope.common.config.FrontendConfig; import com.alibaba.graphscope.common.ir.rel.graph.GraphLogicalSource; +import com.alibaba.graphscope.common.ir.tools.GraphBuilder; import com.alibaba.graphscope.common.ir.tools.LogicalPlan; import com.google.common.collect.ImmutableMap; @@ -504,4 +505,21 @@ public void match_24_test() { + "], matchOpt=[INNER])", node.explain().trim()); } + + @Test + public void property_exist_after_type_inference_test() { + GraphBuilder builder = + com.alibaba.graphscope.common.ir.Utils.mockGraphBuilder( + "schema/ldbc_schema_exp_hierarchy.json"); + // check property 'creationDate' still exists after type inference has updated the type of + // 'HASCREATOR' + RelNode rel = + com.alibaba.graphscope.cypher.antlr4.Utils.eval( + "Match (a:PERSON)<-[h:HASCREATOR]-(b:COMMENT) Return h;", builder) + .build(); + Assert.assertEquals( + "RecordType(Graph_Schema_Type(labels=[EdgeLabel(HASCREATOR, COMMENT, PERSON)]," + + " properties=[BIGINT creationDate]) h)", + rel.getRowType().toString()); + } } diff --git a/interactive_engine/compiler/src/test/resources/proto/intersect_test.json b/interactive_engine/compiler/src/test/resources/proto/intersect_test.json index 5341c37cc6a0..75d2a411c153 100644 --- a/interactive_engine/compiler/src/test/resources/proto/intersect_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/intersect_test.json @@ -86,12 +86,24 @@ "label": { "srcLabel": 2, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }, { "label": { "srcLabel": 3, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }] } }, @@ -250,4 +262,4 @@ } } }] -} \ No newline at end of file +} diff --git a/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json b/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json index 1ca5b9f307de..95399e202519 100644 --- a/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/intersect_test_2.json @@ -73,6 +73,9 @@ "tables": [{ "id": 0 }], + "columns": [{ + "name": "creationDate" + }], "sampleRatio": 1.0 }, "alias": 1, @@ -87,12 +90,24 @@ "label": { "srcLabel": 2, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }, { "label": { "srcLabel": 3, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }] } }, @@ -436,4 +451,4 @@ } } }] -} \ No newline at end of file +} diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json index 015570927444..002f573adf14 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test.json @@ -94,12 +94,24 @@ "label": { "srcLabel": 2, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }, { "label": { "srcLabel": 3, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }] } }, @@ -274,4 +286,4 @@ } } }] -} \ No newline at end of file +} diff --git a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json index 50aa41100031..eacf38f51d36 100644 --- a/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json +++ b/interactive_engine/compiler/src/test/resources/proto/partitioned_intersect_test_2.json @@ -81,6 +81,9 @@ "tables": [{ "id": 0 }], + "columns": [{ + "name": "creationDate" + }], "sampleRatio": 1.0 }, "alias": 1, @@ -95,12 +98,24 @@ "label": { "srcLabel": 2, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }, { "label": { "srcLabel": 3, "dstLabel": 1 - } + }, + "props": [{ + "propId": { + "name": "creationDate" + }, + "type": "INT64" + }] }] } }, @@ -460,4 +475,4 @@ } } }] -} \ No newline at end of file +} diff --git a/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json b/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json index cd01cb223be5..0211c4994efc 100644 --- a/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json +++ b/interactive_engine/compiler/src/test/resources/proto/scan_edge_test.json @@ -8,6 +8,9 @@ "tables": [{ "id": 0 }], + "columns": [{ + "name": "weight" + }], "sampleRatio": 1.0 } } diff --git a/interactive_engine/compiler/src/test/resources/schema/ldbc_schema_exp_hierarchy.json b/interactive_engine/compiler/src/test/resources/schema/ldbc_schema_exp_hierarchy.json index 6594af8d51d7..43034a576ee7 100644 --- a/interactive_engine/compiler/src/test/resources/schema/ldbc_schema_exp_hierarchy.json +++ b/interactive_engine/compiler/src/test/resources/schema/ldbc_schema_exp_hierarchy.json @@ -814,7 +814,16 @@ } } ], - "columns": [] + "columns": [ + { + "key": { + "id": 2, + "name": "creationDate" + }, + "data_type": 2, + "is_primary_key": false + } + ] }, { "label": { diff --git a/interactive_engine/executor/assembly/groot/src/store/graph.rs b/interactive_engine/executor/assembly/groot/src/store/graph.rs index 794ce0ab4f65..4a91c6e34df9 100644 --- a/interactive_engine/executor/assembly/groot/src/store/graph.rs +++ b/interactive_engine/executor/assembly/groot/src/store/graph.rs @@ -528,11 +528,11 @@ pub extern "C" fn tryCatchUpWithPrimary(ptr: GraphHandle) -> Box { // sleep 2 min for the underlying storage catch latest changes. match graph_store_ptr.reopen(120) { Ok(_) => { - info!("Reopened store"); + info!("Reopened store after try catchup with primary"); JnaResponse::new_success() } Err(e) => { - let msg = format!("Reopen failed: {:?}", e); + let msg = format!("Reopen failed after try catchup with primary: {:?}", e); error!("{}", msg); JnaResponse::new_error(&msg) } diff --git a/interactive_engine/executor/engine/pegasus/server/build.rs b/interactive_engine/executor/engine/pegasus/server/build.rs index 841af8d28430..f0d29eb9b76d 100644 --- a/interactive_engine/executor/engine/pegasus/server/build.rs +++ b/interactive_engine/executor/engine/pegasus/server/build.rs @@ -16,6 +16,7 @@ fn main() -> Result<(), Box> { println!("cargo:rerun-if-changed=proto/job_service.proto"); println!("cargo:rerun-if-changed=proto/job_plan.proto"); + println!("cargo:rerun-if-changed=../../../../../proto/error/insight.proto"); codegen_inplace() } @@ -30,7 +31,14 @@ fn codegen_inplace() -> Result<(), Box> { .build_server(true) .build_client(true) .out_dir("src/generated") - .compile(&["proto/job_service.proto", "proto/job_plan.proto"], &["proto"])?; + .compile( + &[ + "proto/job_service.proto", + "proto/job_plan.proto", + "../../../../../proto/error/insight.proto", + ], + &["proto", "../../../../../proto"], + )?; Ok(()) } @@ -39,6 +47,13 @@ fn codegen_inplace() -> Result<(), Box> { tonic_build::configure() .build_server(true) .build_client(true) - .compile(&["proto/job_service.proto", "proto/job_plan.proto"], &["proto"])?; + .compile( + &[ + "proto/job_service.proto", + "proto/job_plan.proto", + "../../../../../proto/error/insight.proto", + ], + &["proto", "../../../../../proto"], + )?; Ok(()) } diff --git a/interactive_engine/executor/engine/pegasus/server/src/error.rs b/interactive_engine/executor/engine/pegasus/server/src/error.rs new file mode 100644 index 000000000000..ad7bfdd43701 --- /dev/null +++ b/interactive_engine/executor/engine/pegasus/server/src/error.rs @@ -0,0 +1,116 @@ +// +//! Copyright 2024 Alibaba Group Holding Limited. +//! +//! Licensed under the Apache License, Version 2.0 (the "License"); +//! you may not use this file except in compliance with the License. +//! You may obtain a copy of the License at +//! +//! http://www.apache.org/licenses/LICENSE-2.0 +//! +//! Unless required by applicable law or agreed to in writing, software +//! distributed under the License is distributed on an "AS IS" BASIS, +//! WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//! See the License for the specific language governing permissions and +//! limitations under the License. + +use std::collections::HashMap; + +use pegasus::{ + errors::{ErrorKind, JobExecError}, + JobSubmitError, +}; + +use crate::insight_error::Code as ErrorCode; + +#[derive(Clone)] +pub struct ServerError { + err_code: ErrorCode, + ec: String, + msg: String, + details: HashMap, +} + +impl ServerError { + pub fn new(err_code: ErrorCode, msg: String) -> Self { + let ec = format!("04-{:04}", err_code as i32); + ServerError { err_code, ec, msg, details: HashMap::new() } + } + + pub fn with_details(mut self, key: &str, value: String) -> Self { + self.details.insert(key.to_string(), value); + self + } +} + +impl std::fmt::Debug for ServerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + writeln!(f, "")?; + writeln!(f, "ErrorCode: {:?}", self.err_code)?; + writeln!(f, "EC: {}", self.ec)?; + writeln!(f, "Message: \"{}\"", self.msg)?; + for (k, v) in self.details.iter() { + writeln!(f, "{}: {}", k, v)?; + } + write!(f, "") + } +} + +impl From<&JobExecError> for ServerError { + fn from(err: &JobExecError) -> Self { + match err.kind { + ErrorKind::WouldBlock(_) => { + let err_code = ErrorCode::JobExecuteWouldBlock; + ServerError::new(err_code, format!("{}", err)) + } + ErrorKind::Interrupted => { + let err_code = ErrorCode::JobExecuteInterrupted; + ServerError::new(err_code, format!("{}", err)) + } + ErrorKind::IOError => { + let err_code = ErrorCode::JobExecuteIoError; + ServerError::new(err_code, format!("{}", err)) + } + ErrorKind::IllegalScopeInput => { + let err_code = ErrorCode::JobExecuteIlleagalScopeInput; + ServerError::new(err_code, format!("{}", err)) + } + ErrorKind::Canceled => { + let err_code = ErrorCode::JobExecuteCancelled; + ServerError::new(err_code, format!("{}", err)) + } + ErrorKind::Others => { + let err_code = ErrorCode::JobExecuteOthers; + ServerError::new(err_code, format!("{}", err)) + } + } + } +} + +impl From<&JobSubmitError> for ServerError { + fn from(err: &JobSubmitError) -> Self { + match err { + JobSubmitError::Build(err) => match err { + pegasus::BuildJobError::Unsupported(e) => { + let err_code = ErrorCode::JobSubmitBuildJobUnsupported; + ServerError::new(err_code, format!("{}", e)) + } + pegasus::BuildJobError::InternalError(e) => { + let err_code = ErrorCode::JobSubmitBuildJobInternalError; + ServerError::new(err_code, format!("{}", e)) + } + pegasus::BuildJobError::ServerError(e) => { + let err_code = ErrorCode::JobSubmitBuildJobServerError; + ServerError::new(err_code, format!("{}", e)) + } + pegasus::BuildJobError::UserError(e) => { + let err_code = ErrorCode::JobSubmitBuildJobUserError; + ServerError::new(err_code, format!("{}", e)) + } + }, + JobSubmitError::Spawn(e) => { + let err_code = ErrorCode::JobSubmitSpawnJobError; + ServerError::new(err_code, format!("{}", e)) + } + } + } +} diff --git a/interactive_engine/executor/engine/pegasus/server/src/lib.rs b/interactive_engine/executor/engine/pegasus/server/src/lib.rs index 3432234ccdd6..6e2a0f6c12b6 100644 --- a/interactive_engine/executor/engine/pegasus/server/src/lib.rs +++ b/interactive_engine/executor/engine/pegasus/server/src/lib.rs @@ -26,6 +26,9 @@ mod generated { pub mod job_proto { tonic::include_proto!("job_proto"); } + pub mod insight_error { + tonic::include_proto!("insight_error"); + } } #[rustfmt::skip] @@ -35,8 +38,11 @@ mod generated { pub mod protocol; #[path = "job_proto.rs"] pub mod job_proto; + #[path = "insight_error.rs"] + pub mod insight_error; } +pub use generated::insight_error; pub use generated::job_proto as job_pb; pub use generated::protocol as pb; @@ -46,6 +52,7 @@ pub trait AnyData: Data + Eq {} pub mod client; pub mod cluster; pub mod config; +mod error; pub mod job; pub mod rpc; diff --git a/interactive_engine/executor/engine/pegasus/server/src/rpc.rs b/interactive_engine/executor/engine/pegasus/server/src/rpc.rs index e4f6c361aae2..cb5698f8247d 100644 --- a/interactive_engine/executor/engine/pegasus/server/src/rpc.rs +++ b/interactive_engine/executor/engine/pegasus/server/src/rpc.rs @@ -34,17 +34,16 @@ use opentelemetry::{ trace::{Span, SpanKind, Tracer}, KeyValue, }; -use opentelemetry_otlp::{ExportConfig, Protocol, TonicExporterBuilder, WithExportConfig}; +use opentelemetry_otlp::{TonicExporterBuilder, WithExportConfig}; use opentelemetry_sdk::metrics::SdkMeterProvider; use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::resource::{ EnvResourceDetector, SdkProvidedResourceDetector, TelemetryResourceDetector, }; -use opentelemetry_sdk::trace::BatchConfigBuilder; use opentelemetry_sdk::Resource; use pegasus::api::function::FnResult; use pegasus::api::FromStream; -use pegasus::errors::{ErrorKind, JobExecError}; +use pegasus::errors::JobExecError; use pegasus::result::{FromStreamExt, ResultSink}; use pegasus::{Configuration, Data, JobConf, ServerConf}; use pegasus_network::config::ServerAddr; @@ -55,6 +54,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use tonic::transport::Server; use tonic::{Code, Request, Response, Status}; +use crate::error::ServerError; use crate::generated::protocol as pb; use crate::generated::protocol::job_config::Servers; use crate::job::{JobAssembly, JobDesc}; @@ -103,24 +103,12 @@ impl FromStreamExt> for RpcSink { fn on_error(&mut self, error: Box) { self.had_error.store(true, Ordering::SeqCst); let status = if let Some(e) = error.downcast_ref::() { - match e.kind { - ErrorKind::WouldBlock(_) => { - Status::internal(format!("[Execution Error] WouldBlock: {}", error)) - } - ErrorKind::Interrupted => { - Status::internal(format!("[Execution Error] Interrupted: {}", error)) - } - ErrorKind::IOError => Status::internal(format!("[Execution Error] IOError: {}", error)), - ErrorKind::IllegalScopeInput => { - Status::internal(format!("[Execution Error] IllegalScopeInput: {}", error)) - } - ErrorKind::Canceled => { - Status::deadline_exceeded(format!("[Execution Error] Canceled: {}", error)) - } - _ => Status::unknown(format!("[Execution Error]: {}", error)), - } + let server_error = ServerError::from(e).with_details("QueryId", self.job_id.to_string()); + Status::internal(format!("{:?}", server_error)) } else { - Status::unknown(format!("[Unknown Error]: {}", error)) + let server_error = + ServerError::new(crate::insight_error::Code::UnknownError, error.to_string()); + Status::unknown(format!("{:?}", server_error)) }; self.tx.send(Err(status)).ok(); @@ -235,7 +223,7 @@ where info!("trace_id : {}, job conf {:?}", trace_id_hex, conf); span.set_attributes(vec![ KeyValue::new("job.name", conf.job_name.clone()), - KeyValue::new("job.id", conf.job_id.to_string()), + KeyValue::new("job.id", job_id.to_string()), ]); let cx = opentelemetry::Context::current_with_span(span); let _guard = cx.clone().attach(); @@ -243,7 +231,10 @@ where if let Err(e) = ret { error!("trace_id:{}, submit job {} failure: {:?}", trace_id_hex, job_id, e); - Err(Status::unknown(format!("submit job error {}", e))) + let server_error = ServerError::from(&e) + .with_details("TraceId", trace_id_hex) + .with_details("QueryId", job_id.to_string()); + Err(Status::internal(format!("{:?}", server_error))) } else { Ok(Response::new(UnboundedReceiverStream::new(rx))) } diff --git a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs index d894ad20c8c6..326b17519c4b 100644 --- a/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs +++ b/interactive_engine/executor/ir/graph_proxy/src/apis/graph/element/path.rs @@ -188,11 +188,12 @@ impl GraphPath { } // append another path to the current path, and return the flag of whether the path has been appended or not. - // notice that, if the path is a simple path, we simply concatenate the two paths, without checking the duplication (this may not be as expected) // e.g., [1,2,3] + [4,5] = [1,2,3,4,5] + // notice that, if the path is a simple path, we further check the duplication + // e.g., [1,2,3] + [3,4,5] will return false pub fn append_path(&mut self, other: GraphPath) -> bool { match self { - GraphPath::AllPath(ref mut p) | GraphPath::SimpleAllPath(ref mut p) => { + GraphPath::AllPath(ref mut p) => { if let Some(other_path) = other.take_path() { p.extend(other_path); true @@ -200,6 +201,18 @@ impl GraphPath { false } } + GraphPath::SimpleAllPath(_) => { + if let Some(other_path) = other.take_path() { + for e in other_path { + if !self.append(e) { + return false; + } + } + true + } else { + false + } + } GraphPath::EndV(_) | GraphPath::SimpleEndV(_) => false, } } diff --git a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs index 7f4ddf896b9b..0bf37fd2b18f 100644 --- a/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs +++ b/interactive_engine/executor/ir/runtime/src/process/operator/map/project.rs @@ -317,7 +317,7 @@ fn exec_projector(input: &Record, projector: &Projector) -> FnExecResult) -> GraphPath { + let details = DynDetails::default(); + let mut path = GraphPath::new( + Vertex::new(vids[0], None, details.clone()), + pb::path_expand::PathOpt::Simple, + pb::path_expand::ResultOpt::AllV, + ); + for i in 1..vids.len() { + path.append(Vertex::new(vids[i], None, details.clone())); + } + path + } + fn build_project_path_concat( left_endpoint: common_pb::path_concat::Endpoint, right_endpoint: common_pb::path_concat::Endpoint, ) -> pb::Project { @@ -1739,6 +1752,40 @@ mod tests { assert_eq!(results, vec![concat_path]); } + #[test] + fn project_concat_simple_path_test_01() { + // sub_path1: [1,2] + let sub_path1 = build_simple_path(vec![1, 2]); + // sub_path2: [3,2] + let sub_path2 = build_simple_path(vec![3, 2]); + // concat project + let project_opr_pb = build_project_path_concat( + common_pb::path_concat::Endpoint::End, + common_pb::path_concat::Endpoint::End, + ); + // concat path: [1,2,3] + let concat_path = build_path(vec![1, 2, 3]); + project_concat_allv_path_test(sub_path1, sub_path2, project_opr_pb, concat_path); + } + + #[test] + fn project_concat_simple_path_test_02() { + // sub_path1: [1,4,2] + let sub_path1 = build_simple_path(vec![1, 4, 2]); + // sub_path2: [3,4,2] + let sub_path2 = build_simple_path(vec![3, 4, 2]); + // concat project + let project_opr_pb = build_project_path_concat( + common_pb::path_concat::Endpoint::End, + common_pb::path_concat::Endpoint::End, + ); + // concat path: None + let mut r1 = Record::new(sub_path1, Some(TAG_A.into())); + r1.append(sub_path2, Some(TAG_B.into())); + let mut result = project_test(vec![r1], project_opr_pb); + assert!(result.next().is_none()); + } + // a fail test case #[test] fn project_concat_allv_path_error_test() { diff --git a/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs b/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs index 50488abb6983..080f213d8293 100644 --- a/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs +++ b/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs @@ -71,6 +71,9 @@ impl RocksDB { } fn replace_db(&self, db: DB) { + // To force any deferred work to run, we need the epoch to move forward two times. + epoch::pin().flush(); + epoch::pin().flush(); let guard = epoch::pin(); let new_db = Arc::new(db); let new_db_shared = Owned::new(new_db).into_shared(&guard); @@ -92,9 +95,6 @@ impl RocksDB { drop(old_db_shared.into_owned()) }) } - // To force any deferred work to run, we need the epoch to move forward two times. - epoch::pin().flush(); - epoch::pin().flush(); info!("RocksDB {:} replaced", path); } diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java index 90e6e500a838..c80465644d8c 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java @@ -56,4 +56,9 @@ public GraphStatistics readStats(GraphId graphId) { throw new UnimplementedException( "reading graph statistics in vineyard is unimplemented yet"); } + + @Override + public boolean syncStatsEnabled(GraphId graphId) throws IOException { + return false; + } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/IngestorWriteService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/IngestorWriteService.java index 209532a866e1..0d81e9d9c65e 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/IngestorWriteService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/IngestorWriteService.java @@ -20,7 +20,11 @@ import io.grpc.stub.StreamObserver; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class IngestorWriteService extends IngestorWriteGrpc.IngestorWriteImplBase { + private static final Logger logger = LoggerFactory.getLogger(KafkaAppender.class); private final KafkaAppender kafkaAppender; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java index 8d5df033a6e1..d46e54aec283 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java @@ -14,6 +14,8 @@ package com.alibaba.graphscope.groot.frontend; import com.alibaba.graphscope.groot.SnapshotWithSchema; +import com.alibaba.graphscope.groot.common.config.CommonConfig; +import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.exception.UnimplementedException; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; @@ -30,20 +32,22 @@ public class WrappedSchemaFetcher implements SchemaFetcher { private final SnapshotCache snapshotCache; private final MetaService metaService; - // If this is a secondary instance, then always use the latest snapshot ID. - private final boolean isSecondary; + private final boolean collectStatistics; public WrappedSchemaFetcher( - SnapshotCache snapshotCache, MetaService metaService, boolean isSecondary) { + SnapshotCache snapshotCache, MetaService metaService, Configs configs) { this.snapshotCache = snapshotCache; this.metaService = metaService; - this.isSecondary = isSecondary; + // If this is a secondary instance, then always use the latest snapshot ID. + // boolean isSecondary = CommonConfig.SECONDARY_INSTANCE_ENABLED.get(configs); + this.collectStatistics = CommonConfig.COLLECT_STATISTICS.get(configs); } @Override public Map getSchemaSnapshotPair() { SnapshotWithSchema snapshotSchema = this.snapshotCache.getSnapshotWithSchema(); long MAX_SNAPSHOT_ID = Long.MAX_VALUE - 1; + // if (isSecondary) {long MAX_SNAPSHOT_ID = Long.MAX_VALUE - 1;} // Always retrieve the latest snapshot id to avoid inconsistency. long snapshotId = MAX_SNAPSHOT_ID; GraphSchema schema = snapshotSchema.getGraphDef(); @@ -64,4 +68,9 @@ public int getPartitionNum() { public int getVersion() { throw new UnimplementedException(); } + + @Override + public boolean statisticsEnabled() { + return collectStatistics; + } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java index 4c0731dadb36..23f607609902 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java @@ -185,7 +185,7 @@ private void addUpdateEdgeOperation( EdgeId edgeId = getEdgeId(schema, dataRecord, false); if (edgeId.id == 0) { // This is for update edge, if edgeInnerId is 0, generate new id, incase there isn't - // such a edge + // such an edge edgeId.id = edgeIdGenerator.getNextId(); } EdgeKind edgeKind = getEdgeKind(schema, dataRecord); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java index 283f0ddd7a97..bb2f848843b5 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java @@ -23,10 +23,7 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.TimeUnit; @@ -43,8 +40,7 @@ public class KafkaAppender { private final int storeCount; private final int partitionCount; - private final int bufferSize; - private BlockingQueue ingestBuffer; + private final BlockingQueue ingestBuffer; private Thread ingestThread; private boolean shouldStop = false; @@ -58,9 +54,9 @@ public KafkaAppender(Configs configs, MetaService metaService, LogService logSer this.queue = CommonConfig.NODE_IDX.get(configs); this.storeCount = CommonConfig.STORE_NODE_COUNT.get(configs); this.partitionCount = metaService.getPartitionCount(); - this.bufferSize = FrontendConfig.WRITE_QUEUE_BUFFER_MAX_COUNT.get(configs); + int bufferSize = FrontendConfig.WRITE_QUEUE_BUFFER_MAX_COUNT.get(configs); this.ingestSnapshotId = new AtomicLong(-1); - this.ingestBuffer = new ArrayBlockingQueue<>(this.bufferSize); + this.ingestBuffer = new ArrayBlockingQueue<>(bufferSize); initMetrics(); } @@ -163,7 +159,8 @@ private long processTask(LogWriter logWriter, IngestTask task) throws IOExceptio for (Map.Entry entry : builderMap.entrySet()) { int storeId = entry.getKey(); OperationBatch batch = entry.getValue().build(); - // logger.info("Log writer append partitionId [{}]", storeId); + // logger.info("Log writer append storeId [{}], batch size: {}", storeId, + // batch.getOperationCount()); logWriter.append(storeId, new LogEntry(ingestSnapshotId.get(), batch)); } } catch (Exception e) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java index b57533049342..aa5e825bd00e 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java @@ -138,6 +138,10 @@ public Builder setTraceId(String traceId) { return this; } + public int getOperationCount() { + return operationBlobs.size(); + } + public OperationBatch build() { this.built = true; return new OperationBatch(latestSnapshotId, operationBlobs, traceId); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java index 8202e39f6a9f..f858bb5da13c 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java @@ -30,9 +30,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.*; import java.util.concurrent.atomic.AtomicReference; public class KafkaProcessor { @@ -47,7 +45,7 @@ public class KafkaProcessor { private final ObjectMapper objectMapper; private ScheduledExecutorService persistOffsetsScheduler; private Thread pollThread; - + private Thread processThread; private final boolean isSecondary; private final long offsetsPersistIntervalMs; private final WriterAgent writerAgent; @@ -58,6 +56,8 @@ public class KafkaProcessor { private volatile boolean shouldStop = true; List typesDDL; + BlockingQueue> queue; + public KafkaProcessor( Configs configs, MetaService metaService, @@ -74,6 +74,9 @@ public KafkaProcessor( storeId = CommonConfig.NODE_IDX.get(configs); offsetsPersistIntervalMs = CoordinatorConfig.OFFSETS_PERSIST_INTERVAL_MS.get(configs); + + int queueSize = StoreConfig.STORE_QUEUE_BUFFER_SIZE.get(configs); + queue = new ArrayBlockingQueue<>(queueSize); } public void start() { @@ -88,13 +91,7 @@ public void start() { ThreadFactoryUtils.daemonThreadFactoryWithLogExceptionHandler( "persist-offsets-scheduler", logger)); this.persistOffsetsScheduler.scheduleWithFixedDelay( - () -> { - try { - updateQueueOffsets(); - } catch (Exception e) { - logger.error("error in updateQueueOffsets, ignore", e); - } - }, + this::updateQueueOffsets, offsetsPersistIntervalMs, offsetsPersistIntervalMs, TimeUnit.MILLISECONDS); @@ -103,20 +100,23 @@ public void start() { this.pollThread.setName("store-kafka-poller"); this.pollThread.setDaemon(true); this.pollThread.start(); + + this.processThread = new Thread(this::processRecords); + this.processThread.setName("store-kafka-record-processor"); + this.processThread.setDaemon(true); + this.processThread.start(); + logger.info("Kafka processor started"); } public void stop() { this.shouldStop = true; - try { - updateQueueOffsets(); - } catch (IOException ex) { - logger.error("update queue offset failed", ex); - } + updateQueueOffsets(); if (this.persistOffsetsScheduler != null) { this.persistOffsetsScheduler.shutdown(); try { - this.persistOffsetsScheduler.awaitTermination(3000, TimeUnit.MILLISECONDS); + boolean ignored = + this.persistOffsetsScheduler.awaitTermination(3000, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { // Ignore } @@ -163,7 +163,7 @@ public void recover() throws IOException { } } - private void updateQueueOffsets() throws IOException { + private void updateQueueOffsets() { List queueOffsets = this.queueOffsetsRef.get(); List newQueueOffsets = new ArrayList<>(queueOffsets); boolean changed = false; @@ -176,8 +176,12 @@ private void updateQueueOffsets() throws IOException { } } if (changed) { - persistObject(newQueueOffsets, QUEUE_OFFSETS_PATH); - this.queueOffsetsRef.set(newQueueOffsets); + try { + persistObject(newQueueOffsets, QUEUE_OFFSETS_PATH); + this.queueOffsetsRef.set(newQueueOffsets); + } catch (IOException e) { + logger.error("error in updateQueueOffsets, ignore", e); + } } } @@ -201,7 +205,7 @@ public void pollBatches() { while (!shouldStop) { ConsumerRecords records = reader.getLatestUpdates(); for (ConsumerRecord record : records) { - processRecord(record); + queue.add(record); } } } catch (IOException e) { @@ -256,20 +260,33 @@ public void replayWAL() throws IOException { long replayFrom = queueOffsetsRef.get().get(0) + 1; logger.info("replay WAL of queue#[{}] from offset [{}]", storeId, replayFrom); if (replayFrom == 0) { - logger.warn("It may not useful to replay from the 0 offset, skipped"); + logger.warn("It may not be expected to replay from the 0 offset, skipped"); return; } int replayCount = 0; try (LogReader logReader = this.logService.createReader(storeId, replayFrom)) { ConsumerRecord record; while ((record = logReader.readNextRecord()) != null) { - processRecord(record); + queue.put(record); replayCount++; } + } catch (InterruptedException e) { + throw new RuntimeException(e); } logger.info("replayWAL finished. total replayed [{}] records", replayCount); } + private void processRecords() { + while (true) { + try { + ConsumerRecord record = queue.take(); + processRecord(record); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + private List prepareDDLTypes() { List types = new ArrayList<>(); types.add(OperationType.CREATE_VERTEX_TYPE); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java index 1afa69a48fa8..136d0639ae61 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java @@ -220,7 +220,6 @@ public void stop() { public boolean batchWrite(StoreDataBatch storeDataBatch) throws ExecutionException, InterruptedException { - long start = System.currentTimeMillis(); long snapshotId = storeDataBatch.getSnapshotId(); List> dataBatch = storeDataBatch.getDataBatch(); AtomicBoolean hasDdl = new AtomicBoolean(false); @@ -268,7 +267,9 @@ private Map writeStore( this.writeHistogram.record( System.currentTimeMillis() - start, attrs.build()); this.writeCounter.add(batch.getOperationCount(), attrs.build()); - } + } // else { + // logger.debug("marker batch ignored"); + // } } catch (Exception ex) { metricLogger.info(buildMetricJsonLog(false, batch, start, partitionId)); logger.error( @@ -298,8 +299,9 @@ private Map writeStore( } future.get(); if (batchNeedRetry.size() > 0) { + logger.warn("Write batch failed, will retry. failure count: {}", batchNeedRetry.size()); try { - Thread.sleep(1000L); + Thread.sleep(100L); } catch (InterruptedException e) { // Ignore } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java index bfaf456df732..11aa9ef522f2 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java @@ -135,7 +135,7 @@ public Frontend(Configs configs) { boolean isSecondary = CommonConfig.SECONDARY_INSTANCE_ENABLED.get(configs); WrappedSchemaFetcher wrappedSchemaFetcher = - new WrappedSchemaFetcher(snapshotCache, metaService, isSecondary); + new WrappedSchemaFetcher(snapshotCache, metaService, configs); IrServiceProducer serviceProducer = new IrServiceProducer(configs); this.graphService = serviceProducer.makeGraphService(wrappedSchemaFetcher, channelManager); } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java index f2cfa8ad2c85..a32b2d6f01e8 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GrootIrMetaReader.java @@ -50,4 +50,9 @@ public IrMeta readMeta() throws IOException { public GraphStatistics readStats(GraphId graphId) throws IOException { return schemaFetcher.getStatistics(); } + + @Override + public boolean syncStatsEnabled(GraphId graphId) { + return schemaFetcher.statisticsEnabled(); + } } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index a8a3da0de562..87a14c0abd35 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -142,6 +142,8 @@ private com.alibaba.graphscope.common.config.Configs getConfigs() { addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_RULES.getKey(), configMap); addToConfigMapIfExist(FrontendConfig.GREMLIN_SCRIPT_LANGUAGE_NAME.getKey(), configMap); addToConfigMapIfExist(FrontendConfig.GRAPH_PHYSICAL_OPT.getKey(), configMap); + addToConfigMapIfExist(PlannerConfig.GRAPH_PLANNER_CBO_GLOGUE_SIZE.getKey(), configMap); + addToConfigMapIfExist(PlannerConfig.JOIN_MIN_PATTERN_SIZE.getKey(), configMap); return new com.alibaba.graphscope.common.config.Configs(configMap); } diff --git a/k8s/Makefile b/k8s/Makefile index 84c829324f30..4dfb2e9fdb47 100644 --- a/k8s/Makefile +++ b/k8s/Makefile @@ -6,8 +6,10 @@ ifeq ($(REGISTRY),) REGISTRY := registry.cn-hongkong.aliyuncs.com endif -# x86_64 or aarch64 -ARCH := $(shell uname -m) +PLATFORM := $(shell uname -m) +# can be: x86_64, arm64 +ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) + VERSION ?= latest VINEYARD_VERSION ?= v0.23.0 @@ -41,36 +43,34 @@ interactive: interactive-frontend interactive-executor all: coordinator analytical interactive learning graphscope: all +manylinux2014: + cd $(WORKING_DIR)/../ && \ + docker build \ + -t graphscope/manylinux2014:${ARCH} \ + -f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile . + dev-wheel: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ - -f $(DOCKERFILES_DIR)/manylinux2014.Dockerfile . - - -manylinux2014-ext: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ - docker build \ - -t graphscope/manylinux2014:ext-${ARCH} \ - -f $(DOCKERFILES_DIR)/manylinux2014-ext.Dockerfile . + -f $(DOCKERFILES_DIR)/graphscope-dev-wheel.Dockerfile . graphscope-dev: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/graphscope-dev:${VINEYARD_VERSION}-${ARCH} \ -f $(DOCKERFILES_DIR)/graphscope-dev.Dockerfile . vineyard-dev: - cd $(WORKING_DIR) && \ - cd ../python/graphscope && \ + cd $(WORKING_DIR)/../ && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg VINEYARD_VERSION=${VINEYARD_VERSION} \ -t graphscope/vineyard-dev:${VINEYARD_VERSION}-${ARCH} \ @@ -79,6 +79,7 @@ vineyard-dev: vineyard-runtime: cd $(WORKING_DIR) && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ -t graphscope/vineyard-runtime:${VINEYARD_VERSION}-${ARCH} \ @@ -88,6 +89,7 @@ coordinator: cd $(WORKING_DIR)/.. && \ docker build \ --target coordinator \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg CI=${CI} \ @@ -98,6 +100,7 @@ analytical: cd $(WORKING_DIR)/.. && \ docker build \ --target analytical \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -109,6 +112,7 @@ analytical-java: cd $(WORKING_DIR)/.. && \ docker build \ --target analytical-java \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -120,6 +124,7 @@ interactive-frontend: cd $(WORKING_DIR)/.. && \ docker build \ --target frontend \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -132,6 +137,7 @@ interactive-executor: cd $(WORKING_DIR)/.. \ && docker build \ --target executor \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -145,6 +151,7 @@ interactive-experimental: cd $(WORKING_DIR)/.. && \ docker build \ --target experimental \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ -t graphscope/interactive-experimental:${VERSION} \ @@ -154,18 +161,20 @@ interactive-experimental: flex-interactive: cd $(WORKING_DIR)/.. && \ docker build \ - -f ${DOCKERFILES_DIR}/flex-interactive.Dockerfile \ --target runtime \ + --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ + --build-arg PLATFORM=${PLATFORM} \ --build-arg ARCH=${ARCH} \ - --build-arg ENABLE_COORDINATOR=${ENABLE_COORDINATOR} \ - -t graphscope/interactive:${VERSION} . + -t graphscope/interactive:${VERSION} \ + -f ${DOCKERFILES_DIR}/flex-interactive.Dockerfile . learning: cd $(WORKING_DIR)/.. && \ docker build \ --target learning \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -177,6 +186,7 @@ graphlearn-torch: cd $(WORKING_DIR)/.. && \ docker build \ --target graphlearn-torch \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg RUNTIME_VERSION=$(RUNTIME_VERSION) \ @@ -187,6 +197,7 @@ graphlearn-torch: graphscope-store: cd $(WORKING_DIR)/.. && \ docker build \ + --build-arg ARCH=$(ARCH) \ --build-arg REGISTRY=$(REGISTRY) \ --build-arg BUILDER_VERSION=$(BUILDER_VERSION) \ --build-arg profile=${PROFILE} \ diff --git a/k8s/dockerfiles/analytical.Dockerfile b/k8s/dockerfiles/analytical.Dockerfile index 9cc0614861aa..c2cfc8a248b9 100644 --- a/k8s/dockerfiles/analytical.Dockerfile +++ b/k8s/dockerfiles/analytical.Dockerfile @@ -1,10 +1,11 @@ # Analytical engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest ############### BUILDER: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false @@ -26,7 +27,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: ANALYTICAL ####################### -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION AS analytical +FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical ENV GRAPHSCOPE_HOME=/opt/graphscope ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$GRAPHSCOPE_HOME/lib @@ -58,7 +59,7 @@ COPY ./k8s/dockerfiles/entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] ############### BUILDER: ANALYTICAL-JAVA ####################### -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder-java +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder-java COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -81,7 +82,7 @@ RUN cd /home/graphscope/GraphScope/ && \ FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm -FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION AS analytical-java +FROM $REGISTRY/graphscope/vineyard-dev:$RUNTIME_VERSION-$ARCH AS analytical-java COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 ENV LLVM11_HOME=/opt/llvm11 ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config diff --git a/k8s/dockerfiles/coordinator.Dockerfile b/k8s/dockerfiles/coordinator.Dockerfile index a2719ce6e68f..2eff6059a2cf 100644 --- a/k8s/dockerfiles/coordinator.Dockerfile +++ b/k8s/dockerfiles/coordinator.Dockerfile @@ -1,8 +1,9 @@ # Coordinator of graphscope engines +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false diff --git a/k8s/dockerfiles/flex-interactive.Dockerfile b/k8s/dockerfiles/flex-interactive.Dockerfile index 641a16539020..ed852bdc4f5b 100644 --- a/k8s/dockerfiles/flex-interactive.Dockerfile +++ b/k8s/dockerfiles/flex-interactive.Dockerfile @@ -1,10 +1,11 @@ # Coordinator of graphscope engines +ARG PLATFORM=x86_64 +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -ARG ARCH=x86_64 +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG ENABLE_COORDINATOR="false" -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder RUN sudo mkdir -p /opt/flex && sudo chown -R graphscope:graphscope /opt/flex/ USER graphscope @@ -53,6 +54,9 @@ RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ python3 -m pip install --upgrade pip && python3 -m pip install -r requirements.txt && \ python3 setup.py build_proto && python3 setup.py bdist_wheel && \ mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/ && \ + cd ${HOME}/GraphScope/python && \ + export WITHOUT_LEARNING_ENGINE=ON && python3 setup.py bdist_wheel && \ + mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/ && \ cd ${HOME}/GraphScope/coordinator && \ python3 setup.py bdist_wheel && \ mkdir -p /opt/flex/wheel && cp dist/*.whl /opt/flex/wheel/; \ @@ -62,7 +66,7 @@ RUN if [ "${ENABLE_COORDINATOR}" = "true" ]; then \ ########################### RUNTIME IMAGE ########################### from ubuntu:22.04 as runtime -ARG ARCH +ARG PLATFORM=x86_64 ARG ENABLE_COORDINATOR="false" ENV DEBIAN_FRONTEND=noninteractive @@ -89,9 +93,13 @@ ENV LC_ALL en_US.UTF-8 ENV TZ=Asia/Shanghai RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +RUN mkdir /opt/vineyard/ + # copy builder's /opt/flex to final image COPY --from=builder /opt/flex /opt/flex COPY --from=builder /opt/graphscope/lib/libgrape-lite.so /opt/flex/lib/ +COPY --from=builder /opt/graphscope/include/ /opt/flex/include/ +COPY --from=builder /opt/vineyard/include/ /opt/vineyard/include/ # copy the builtin graph, modern_graph RUN mkdir -p /opt/flex/share/gs_interactive_default_graph/ @@ -106,42 +114,42 @@ RUN sed -i 's/default_graph: modern_graph/default_graph: gs_interactive_default_ # remove bin/run_app RUN rm -rf /opt/flex/bin/run_app -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libsnappy*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libsnappy*.so* /usr/lib/$PLATFORM-linux-gnu/ COPY --from=builder /usr/include/arrow /usr/include/arrow COPY --from=builder /usr/include/yaml-cpp /usr/include/yaml-cpp -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libgflags*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libglog*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libyaml-cpp*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libmpi*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_program_options*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_filesystem*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libboost_thread*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libcrypto*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libopen-rte*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libhwloc*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libgflags*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libglog*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libyaml-cpp*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libmpi*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_program_options*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_filesystem*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libboost_thread*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libcrypto*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libopen-rte*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libhwloc*.so* /usr/lib/$PLATFORM-linux-gnu/ # libunwind for arm64 seems not installed here, and seems not needed for aarch64(tested) -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libunwind*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libarrow.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libopen-pal*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libltdl*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libevent*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libutf8proc*.so* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libre2*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libunwind*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libarrow.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libopen-pal*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libltdl*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libevent*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libutf8proc*.so* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libre2*.so* /usr/lib/$PLATFORM-linux-gnu/ COPY --from=builder /usr/include/glog /usr/include/glog COPY --from=builder /usr/include/gflags /usr/include/gflags -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libprotobuf* /usr/lib/$ARCH-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/libfmt*.so* /usr/lib/$ARCH-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libprotobuf* /usr/lib/$PLATFORM-linux-gnu/ +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/libfmt*.so* /usr/lib/$PLATFORM-linux-gnu/ -COPY --from=builder /usr/lib/$ARCH-linux-gnu/openmpi/include/ /opt/flex/include +COPY --from=builder /usr/lib/$PLATFORM-linux-gnu/openmpi/include/ /opt/flex/include COPY --from=builder /usr/include/boost /usr/include/boost COPY --from=builder /usr/include/google /usr/include/google COPY --from=builder /usr/include/yaml-cpp /usr/include/yaml-cpp -RUN sudo rm -rf /usr/lib/$ARCH-linux-gnu/libLLVM*.so* && sudo rm -rf /opt/flex/lib/libseastar.a && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libcuda.so && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libcudart.so && \ - sudo rm -rf /usr/lib/$ARCH-linux-gnu/lib/libicudata.so* +RUN sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/libLLVM*.so* && sudo rm -rf /opt/flex/lib/libseastar.a && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libcuda.so && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libcudart.so && \ + sudo rm -rf /usr/lib/$PLATFORM-linux-gnu/lib/libicudata.so* RUN sudo ln -sf /opt/flex/bin/* /usr/local/bin/ \ && sudo ln -sfn /opt/flex/include/* /usr/local/include/ \ diff --git a/k8s/dockerfiles/graphlearn-torch.Dockerfile b/k8s/dockerfiles/graphlearn-torch.Dockerfile index 8a763a3499ba..970eebc2282e 100644 --- a/k8s/dockerfiles/graphlearn-torch.Dockerfile +++ b/k8s/dockerfiles/graphlearn-torch.Dockerfile @@ -1,9 +1,10 @@ # Graphlearn-torch engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=v0.21.3 ARG RUNTIME_VERSION=v0.21.3 -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope @@ -24,7 +25,7 @@ RUN cd /home/graphscope/GraphScope/; \ cp dist/*.whl /home/graphscope/install/ ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS graphlearn-torch +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS graphlearn-torch RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile new file mode 100644 index 000000000000..1a58a42839f8 --- /dev/null +++ b/k8s/dockerfiles/graphscope-dev-wheel.Dockerfile @@ -0,0 +1,77 @@ +# build `graphscope/graphscope-dev:wheel-{v6d_version}-{arch}` image based on manylinux, +# including all dependencies for building graphscope wheel package. + +ARG ARCH=amd64 +ARG REGISTRY=registry.cn-hongkong.aliyuncs.com +FROM $REGISTRY/graphscope/manylinux2014:$ARCH AS builder + +# build form https://github.com/sighingnow/manylinux/tree/dyn-rebase +# usually we don't need to change this image unless the underlying python needs to be updated +FROM $REGISTRY/graphscope/manylinux2014:20230407 + +# change the source +RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^#.*baseurl=http/baseurl=http/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^mirrorlist=http/#mirrorlist=http/g" /etc/yum.repos.d/*.repo + +# shanghai zoneinfo +ENV TZ=Asia/Shanghai +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo '$TZ' > /etc/timezone + +# for programming output +RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 + +ENV GRAPHSCOPE_HOME=/opt/graphscope + +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$GRAPHSCOPE_HOME/lib:$GRAPHSCOPE_HOME/lib64 +ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:/home/graphscope/.local/bin:/home/graphscope/.cargo/bin + +ENV JAVA_HOME=/usr/lib/jvm/java +ENV RUST_BACKTRACE=1 + +# install clang-11 with gold optimizer plugin, depends on header include/plugin-api.h +# COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 +# ENV LLVM11_HOME=/opt/llvm11 +# ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config + +# Copy the thirdparty c++ dependencies, maven, and hadoop +COPY --from=builder /opt/graphscope /opt/graphscope +COPY --from=builder /opt/openmpi /opt/openmpi +RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* + +RUN useradd -m graphscope -u 1001 \ + && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +# Install jdk-11 +RUN yum install -y sudo vim && \ + yum install python3-pip -y && \ + yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ + yum install java-11-openjdk-devel -y && \ + yum clean all -y --enablerepo='*' && \ + rm -rf /var/cache/yum + +RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /opt/graphscope /opt/vineyard + +USER graphscope +WORKDIR /home/graphscope + +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope +ARG VINEYARD_VERSION=main +RUN cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging --user && \ + python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ + sudo rm -rf /home/graphscope/GraphScope && \ + sudo yum clean all -y && \ + sudo rm -fr /var/cache/yum +RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc + +SHELL [ "/usr/bin/scl", "enable", "rh-python38" ] + +RUN python3 -m pip --no-cache install pyyaml --user +# Uncomment this line will results in a weird error when using the image together with commands, like +# docker run --rm graphscope/graphscope-dev:latest bash -c 'echo xxx && ls -la' +# The output of `ls -la` would not be shown. +# ENTRYPOINT ["/bin/bash", "-c", ". scl_source enable devtoolset-8 rh-python38 && $0 $@"] diff --git a/k8s/dockerfiles/graphscope-dev.Dockerfile b/k8s/dockerfiles/graphscope-dev.Dockerfile index b3df8bef7c6d..1c13a487033d 100644 --- a/k8s/dockerfiles/graphscope-dev.Dockerfile +++ b/k8s/dockerfiles/graphscope-dev.Dockerfile @@ -27,13 +27,13 @@ RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /op USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ +RUN cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION --cn -j $(nproc) && \ cd /home/graphscope && \ - rm -fr gsctl + rm -fr GraphScope RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc RUN python3 -m pip --no-cache install pyyaml ipython --user diff --git a/k8s/dockerfiles/graphscope-store.Dockerfile b/k8s/dockerfiles/graphscope-store.Dockerfile index d8156e50e3b9..19b694b3f085 100644 --- a/k8s/dockerfiles/graphscope-store.Dockerfile +++ b/k8s/dockerfiles/graphscope-store.Dockerfile @@ -1,6 +1,7 @@ +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION as builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH as builder ARG CI=false ARG ENABLE_COORDINATOR=false diff --git a/k8s/dockerfiles/interactive-experimental.Dockerfile b/k8s/dockerfiles/interactive-experimental.Dockerfile index a1cf940bd7eb..478fa3bf60e5 100644 --- a/k8s/dockerfiles/interactive-experimental.Dockerfile +++ b/k8s/dockerfiles/interactive-experimental.Dockerfile @@ -1,8 +1,9 @@ # Interactive engine which uses experimental storage +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope diff --git a/k8s/dockerfiles/interactive.Dockerfile b/k8s/dockerfiles/interactive.Dockerfile index bbb8c305679d..5ab58e8bd340 100644 --- a/k8s/dockerfiles/interactive.Dockerfile +++ b/k8s/dockerfiles/interactive.Dockerfile @@ -1,9 +1,10 @@ # Interactive engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest -FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/graphscope-dev:$BUILDER_VERSION-$ARCH AS builder ARG CI=false @@ -52,8 +53,8 @@ USER graphscope WORKDIR /home/graphscope ############### RUNTIME: executor ####################### -FROM registry.cn-hongkong.aliyuncs.com/graphscope/manylinux2014:ext AS ext -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS executor +FROM registry.cn-hongkong.aliyuncs.com/graphscope/manylinux2014:$ARCH AS ext +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS executor ENV RUST_BACKTRACE=1 diff --git a/k8s/dockerfiles/learning.Dockerfile b/k8s/dockerfiles/learning.Dockerfile index 7edbffa86e80..20b55f08fd67 100644 --- a/k8s/dockerfiles/learning.Dockerfile +++ b/k8s/dockerfiles/learning.Dockerfile @@ -1,5 +1,6 @@ # Learning engine +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest ARG RUNTIME_VERSION=latest @@ -29,7 +30,7 @@ RUN cd /home/graphscope/GraphScope/ && \ fi ############### RUNTIME: GLE ####################### -FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION AS learning +FROM $REGISTRY/graphscope/vineyard-runtime:$RUNTIME_VERSION-$ARCH AS learning RUN sudo apt-get update -y && \ sudo apt-get install -y python3-pip && \ diff --git a/k8s/dockerfiles/manylinux2014-ext.Dockerfile b/k8s/dockerfiles/manylinux2014-ext.Dockerfile deleted file mode 100644 index 80fbbeeed3c6..000000000000 --- a/k8s/dockerfiles/manylinux2014-ext.Dockerfile +++ /dev/null @@ -1,41 +0,0 @@ -# the graphscope-dev-base image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. - -FROM centos:7 AS builder - - -# shanghai zoneinfo -ENV TZ=Asia/Shanghai -RUN yum install sudo -y && \ - yum update glibc-common -y && \ - sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ - yum install python3-pip -y && \ - ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo '$TZ' > /etc/timezone - -ENV LC_ALL=en_US.utf-8 -ENV LANG=en_US.utf-8 - -COPY gsctl ./gsctl -RUN cd ./gsctl && \ - python3 -m pip install click && \ - python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ - rm -fr /root/gsctl - -# install hadoop for processing hadoop data source -RUN if [ "$(uname -m)" = "aarch64" ]; then \ - curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz; \ - tar xzf hadoop-3.3.0-aarch64.tar.gz -C /opt/; \ - else \ - curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz; \ - tar xzf hadoop-3.3.0.tar.gz -C /opt/; \ - fi && \ - rm -rf hadoop-3.3.0* && \ - cd /opt/hadoop-3.3.0/share/ && \ - rm -rf doc hadoop/client hadoop/mapreduce hadoop/tools hadoop/yarn - -FROM centos:7 - -COPY --from=builder /opt/graphscope /opt/graphscope -COPY --from=builder /opt/openmpi /opt/openmpi -COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/manylinux2014.Dockerfile b/k8s/dockerfiles/manylinux2014.Dockerfile index c8ba01c6930d..fe2e85349658 100644 --- a/k8s/dockerfiles/manylinux2014.Dockerfile +++ b/k8s/dockerfiles/manylinux2014.Dockerfile @@ -1,73 +1,45 @@ -# the manylinux2014 image is based on manylinux2014, including all necessary -# dependencies except vineyard for graphscope's wheel package. -# It's tagged as the graphscope/graphscope-dev:wheel +# build `graphscope/manylinux2014` image based on centos7, including all necessary +# dependencies except vineyard +FROM centos:7 AS builder -ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -#FROM vineyardcloudnative/manylinux-llvm:2014-11.0.0 AS llvm -FROM $REGISTRY/graphscope/manylinux2014:ext AS ext - -# build form https://github.com/sighingnow/manylinux/tree/dyn-rebase -# usually we don't need to change this image unless the underlying python needs to be updated -FROM $REGISTRY/graphscope/manylinux2014:20230407 +# change the source +RUN sed -i "s/mirror.centos.org/vault.centos.org/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^#.*baseurl=http/baseurl=http/g" /etc/yum.repos.d/*.repo && \ + sed -i "s/^mirrorlist=http/#mirrorlist=http/g" /etc/yum.repos.d/*.repo # shanghai zoneinfo ENV TZ=Asia/Shanghai -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ - echo '$TZ' > /etc/timezone - -# for programming output -RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 - -ENV GRAPHSCOPE_HOME=/opt/graphscope - -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib:/usr/local/lib64 -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$GRAPHSCOPE_HOME/lib:$GRAPHSCOPE_HOME/lib64 -ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:/home/graphscope/.local/bin:/home/graphscope/.cargo/bin - -ENV JAVA_HOME=/usr/lib/jvm/java -ENV RUST_BACKTRACE=1 - -# install clang-11 with gold optimizer plugin, depends on header include/plugin-api.h -# COPY --from=llvm /opt/llvm11.0.0 /opt/llvm11 -# ENV LLVM11_HOME=/opt/llvm11 -# ENV LIBCLANG_PATH=$LLVM11_HOME/lib LLVM_CONFIG_PATH=$LLVM11_HOME/bin/llvm-config - -# Copy the thirdparty c++ dependencies, maven, and hadoop -COPY --from=ext /opt/graphscope /opt/graphscope -COPY --from=ext /opt/openmpi /opt/openmpi -RUN chmod +x /opt/graphscope/bin/* /opt/openmpi/bin/* - -RUN useradd -m graphscope -u 1001 \ - && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -# Install jdk-11 -RUN yum install -y sudo vim && \ +RUN yum install sudo -y && \ + yum update glibc-common -y && \ + sudo localedef -i en_US -f UTF-8 en_US.UTF-8 && \ yum install python3-pip -y && \ - yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ - yum install java-11-openjdk-devel -y && \ - yum clean all -y --enablerepo='*' && \ - rm -rf /var/cache/yum - -RUN mkdir -p /opt/graphscope /opt/vineyard && chown -R graphscope:graphscope /opt/graphscope /opt/vineyard - -USER graphscope -WORKDIR /home/graphscope - -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl -ARG VINEYARD_VERSION=main -RUN cd /home/graphscope/gsctl && \ - sudo python3 -m pip install click && \ - python3 gsctl.py install-deps dev --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl && \ - sudo yum clean all -y && \ - sudo rm -fr /var/cache/yum -RUN echo ". /home/graphscope/.graphscope_env" >> ~/.bashrc - -SHELL [ "/usr/bin/scl", "enable", "rh-python38" ] + ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && \ + echo '$TZ' > /etc/timezone -RUN python3 -m pip --no-cache install pyyaml --user -# Uncomment this line will results in a weird error when using the image together with commands, like -# docker run --rm graphscope/graphscope-dev:latest bash -c 'echo xxx && ls -la' -# The output of `ls -la` would not be shown. -# ENTRYPOINT ["/bin/bash", "-c", ". scl_source enable devtoolset-8 rh-python38 && $0 $@"] +ENV LC_ALL=en_US.utf-8 +ENV LANG=en_US.utf-8 + +COPY . /root/GraphScope +RUN cd /root/GraphScope && \ + python3 -m pip install click packaging && \ + python3 gsctl.py install-deps dev --cn --for-analytical --no-v6d -j $(nproc) && \ + rm -fr /root/GraphScope + +# install hadoop for processing hadoop data source +RUN if [ "$(uname -m)" = "aarch64" ] || [ "$(uname -m)" = "arm64" ]; then \ + curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0-aarch64.tar.gz; \ + tar xzf hadoop-3.3.0-aarch64.tar.gz -C /opt/; \ + else \ + curl -sS -LO https://archive.apache.org/dist/hadoop/common/hadoop-3.3.0/hadoop-3.3.0.tar.gz; \ + tar xzf hadoop-3.3.0.tar.gz -C /opt/; \ + fi && \ + rm -rf hadoop-3.3.0* && \ + cd /opt/hadoop-3.3.0/share/ && \ + rm -rf doc hadoop/client hadoop/mapreduce hadoop/tools hadoop/yarn + + +FROM centos:7 + +COPY --from=builder /opt/graphscope /opt/graphscope +COPY --from=builder /opt/openmpi /opt/openmpi +COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 diff --git a/k8s/dockerfiles/vineyard-dev.Dockerfile b/k8s/dockerfiles/vineyard-dev.Dockerfile index 7ef99f9a1a53..a68df0989273 100644 --- a/k8s/dockerfiles/vineyard-dev.Dockerfile +++ b/k8s/dockerfiles/vineyard-dev.Dockerfile @@ -1,7 +1,8 @@ # The vineyard-dev image including all vineyard-related dependencies # that could compile graphscope analytical engine. +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com -FROM $REGISTRY/graphscope/manylinux2014:ext AS ext +FROM $REGISTRY/graphscope/manylinux2014:$ARCH AS builder FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive @@ -21,7 +22,7 @@ ENV HADOOP_YARN_HOME=$HADOOP_HOME HADOOP_MAPRED_HOME=$HADOOP_HOME ENV PATH=$PATH:$GRAPHSCOPE_HOME/bin:$HADOOP_HOME/bin:/home/graphscope/.local/bin # Copy hadoop -COPY --from=ext /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 +COPY --from=builder /opt/hadoop-3.3.0 /opt/hadoop-3.3.0 RUN apt-get update && \ apt-get install -y sudo default-jre python3-pip tzdata openssh-server dnsutils && \ @@ -38,13 +39,13 @@ RUN mkdir -p /var/log/graphscope && chown -R graphscope:graphscope /var/log/grap USER graphscope WORKDIR /home/graphscope -COPY --chown=graphscope:graphscope gsctl /home/graphscope/gsctl +COPY --chown=graphscope:graphscope . /home/graphscope/GraphScope ARG VINEYARD_VERSION=main RUN sudo chmod a+wrx /tmp && \ - cd /home/graphscope/gsctl && \ - python3 -m pip install click && \ + cd /home/graphscope/GraphScope && \ + python3 -m pip install click packaging && \ python3 gsctl.py install-deps dev --for-analytical --v6d-version=$VINEYARD_VERSION -j $(nproc) && \ - cd /home/graphscope && sudo rm -rf /home/graphscope/gsctl + cd /home/graphscope && sudo rm -rf /home/graphscope/GraphScope RUN python3 -m pip --no-cache install pyyaml --user diff --git a/k8s/dockerfiles/vineyard-runtime.Dockerfile b/k8s/dockerfiles/vineyard-runtime.Dockerfile index 2c3d0c0cb598..7bcd791fe762 100644 --- a/k8s/dockerfiles/vineyard-runtime.Dockerfile +++ b/k8s/dockerfiles/vineyard-runtime.Dockerfile @@ -1,9 +1,10 @@ # The vineyard-runtime image including all vineyard-related # dependencies that could graphscope interactive engine. +ARG ARCH=amd64 ARG REGISTRY=registry.cn-hongkong.aliyuncs.com ARG BUILDER_VERSION=latest -FROM $REGISTRY/graphscope/vineyard-dev:$BUILDER_VERSION AS builder +FROM $REGISTRY/graphscope/vineyard-dev:$BUILDER_VERSION-$ARCH AS builder FROM ubuntu:22.04 AS runtime @@ -32,7 +33,7 @@ RUN apt-get update -y && \ -P /tmp/ --no-check-certificate && \ sudo apt-get install -y -V /tmp/apache-arrow-apt-source-latest-jammy.deb && \ sudo apt-get update -y && \ - sudo apt-get install -y libarrow-dev libarrow-dataset-dev libarrow-acero-dev libparquet-dev && \ + sudo apt-get install -y libarrow-dev=15.0.2-1 libarrow-dataset-dev=15.0.2-1 libarrow-acero-dev=15.0.2-1 libparquet-dev=15.0.2-1 && \ rm /tmp/apache-arrow-apt-source-latest-*.deb && \ apt-get clean -y && \ rm -rf /var/lib/apt/lists/* diff --git a/k8s/internal/Makefile b/k8s/internal/Makefile index 0260da202275..a3025067e5f7 100644 --- a/k8s/internal/Makefile +++ b/k8s/internal/Makefile @@ -17,8 +17,9 @@ ifeq ($(UNAME_S), Linux) XARGS_EMPTY_FLAG := --no-run-if-empty endif -# x86_64 or arm64 -ARCH := $(shell uname -m) +PLATFORM := $(shell uname -m) +# can be: x86_64, arm64 +ARCH := $(subst x86_64,amd64,$(subst aarch64,arm64,$(PLATFORM))) MACOS_WHEEL_VERSION := 12_0 # docker build arguments @@ -55,7 +56,7 @@ graphscope: graphscope-jupyter-manylinux2014-py3: docker run --rm -it -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c 'pip3 install graphscope-client -U && \ cd /work/python/jupyter/graphscope && \ python3 setup.py bdist_wheel' @@ -81,21 +82,21 @@ graphscope-darwin-py3: # build and delocate wheel cd $(WORKING_DIR)/../../coordinator && \ export WITH_EXTRA_DATA=ON && \ - if [[ "${ARCH}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \ + if [[ "${PLATFORM}" == "arm64" ]]; then export GRAPHSCOPE_HOME=/opt/homebrew; else export GRAPHSCOPE_HOME=/usr/local; fi && \ rm -rf build dist/*.whl || true && \ sudo strip -s $(WORKING_DIR)/../../analytical_engine/exported_symbols_osx.lds ${INSTALL_PREFIX}/bin/grape_engine || true && \ sudo strip ${INSTALL_PREFIX}/bin/gaia_executor && \ export DYLD_LIBRARY_PATH=/usr/local/lib:$$DYLD_LIBRARY_PATH && \ install_name_tool -add_rpath /usr/local/lib ${INSTALL_PREFIX}/bin/gaia_executor && \ - package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-include python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-apps python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-engine python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=gs-coordinator python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ rm -rf build && \ - package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + package_name=graphscope python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ pip3 install delocate==0.10.7 && \ for wheel in `ls dist/*.whl`; do \ delocate-listdeps -a -d $$wheel; \ @@ -109,8 +110,8 @@ graphscope-manylinux2014-py3-nodocker: sudo yum install java-11-openjdk-devel -y && \ sudo yum remove java-1.8.0-openjdk-devel java-1.8.0-openjdk java-1.8.0-openjdk-headless -y && \ cd $(WORKING_DIR)/../.. && \ - if [[ "${ARCH}" == "aarch64" ]]; then \ - export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ + export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ fi && \ @@ -143,13 +144,13 @@ graphscope-manylinux2014-py3-nodocker: package_name=graphscope python3 setup.py bdist_wheel && \ cd dist && \ for wheel in `ls ./*.whl`; do \ - (auditwheel repair $$wheel --plat=manylinux2014_${ARCH} && rm $$wheel) || true; \ + (auditwheel repair $$wheel --plat=manylinux2014_${PLATFORM} && rm $$wheel) || true; \ done graphscope-manylinux2014-py3: - docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} + docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} docker run --rm -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c '. ~/.graphscope_env && \ sudo chown -R graphscope /work && \ cd /work/k8s/internal && \ @@ -170,14 +171,14 @@ graphscope-client-manylinux2014-py3-nodocker: cmake -DKNN=OFF -DWITH_VINEYARD=ON -DTESTING=OFF .. && \ make graphlearn_shared -j`nproc` && \ export LD_LIBRARY_PATH=$(WORKING_DIR)/../../learning_engine/graph-learn/graphlearn/built/lib:$$LD_LIBRARY_PATH && \ - if [[ "${ARCH}" == "aarch64" ]]; then export AUDITWHEEL_PLAT=manylinux2014_${ARCH}; fi && \ + if [[ "${PLATFORM}" == "aarch64" ]]; then export AUDITWHEEL_PLAT=manylinux2014_${PLATFORM}; fi && \ for py in cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310 cp311-cp311; do \ cd $(WORKING_DIR)/../../python; \ export PATH=/opt/python/$$py/bin:$$PATH; \ python3 -m pip install ${PIP_ARGS} -U pip; \ python3 -m pip install ${PIP_ARGS} "torch<=2.2.1" --index-url https://download.pytorch.org/whl/cpu --user; \ if [[ "$$py" == "cp311-cp311" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -185,7 +186,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.23.2" "pandas" wheel "auditwheel==5.2.0"; \ elif [[ "$$py" == "cp310-cp310" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -193,7 +194,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.21.2" "pandas" wheel "auditwheel==5.2.0"; \ elif [[ "$$py" == "cp39-cp39" ]]; then \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -201,7 +202,7 @@ graphscope-client-manylinux2014-py3-nodocker: fi; \ python3 -m pip install ${PIP_ARGS} mypy-protobuf "numpy==1.19.3" "pandas" wheel "auditwheel==5.2.0"; \ else \ - if [[ "${ARCH}" == "aarch64" ]]; then \ + if [[ "${PLATFORM}" == "aarch64" ]]; then \ python3 -m pip install ${PIP_ARGS} grpcio==1.49.1 --no-binary grpcio; \ python3 -m pip install ${PIP_ARGS} grpcio-tools==1.49.1 --no-binary grpcio-tools; \ else \ @@ -214,7 +215,7 @@ graphscope-client-manylinux2014-py3-nodocker: python3 setup.py build_gltorch_ext; \ python3 setup.py bdist_wheel; \ cd dist; \ - auditwheel repair ./*.whl --plat=manylinux2014_${ARCH} --exclude libtorch_cpu.so --exclude libc10.so --exclude libtorch_python.so --exclude libtorch.so; \ + auditwheel repair ./*.whl --plat=manylinux2014_${PLATFORM} --exclude libtorch_cpu.so --exclude libc10.so --exclude libtorch_python.so --exclude libtorch.so; \ done graphscope-client-darwin-py3: @@ -241,29 +242,29 @@ graphscope-client-darwin-py3: elif [[ "$$py" == "310" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.2" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ elif [[ "$$py" == "39" ]]; then \ - if [[ "${ARCH}" == "arm64" ]]; then \ + if [[ "${PLATFORM}" == "arm64" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ else \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.19.3" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ fi; \ else \ - if [[ "${ARCH}" == "arm64" ]]; then \ + if [[ "${PLATFORM}" == "arm64" ]]; then \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.21.0" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ else \ pip3 install ${PIP_ARGS} mypy-protobuf "numpy==1.18.5" "pandas" "grpcio>=1.49" "grpcio-tools>=1.49" delocate wheel; \ fi; \ fi; \ rm -rf build dist/*.whl || true && \ - python3 setup.py build_gltorch_ext --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ - python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${ARCH} && \ + python3 setup.py build_gltorch_ext --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ + python3 setup.py bdist_wheel --plat=macosx_${MACOS_WHEEL_VERSION}_${PLATFORM} && \ for wheel in `ls dist/*.whl`; do \ delocate-wheel -e libtorch_cpu.dylib -e libc10.dylib -e libtorch_python.dylib -e libtorch.dylib -w dist/wheelhouse -v $$wheel && rm $$wheel; \ done graphscope-client-manylinux2014-py3: - docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} + docker pull ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} docker run --rm -v $(WORKING_DIR)/../..:/work \ - ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION} \ + ${REGISTRY}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-${ARCH} \ bash -c '. ~/.graphscope_env && \ sudo chown -R graphscope /work && \ cd /work/k8s/internal && \ diff --git a/proto/error/insight.proto b/proto/error/insight.proto index 8bb83c7360d9..8c91e6256432 100644 --- a/proto/error/insight.proto +++ b/proto/error/insight.proto @@ -23,101 +23,104 @@ limitations under the License. ============================================================================== */ -syntax = "proto3"; + syntax = "proto3"; + package insight_error; + option java_package = "com.alibaba.graphscope.proto"; + option java_multiple_files = true; + + // component-04: GIE Insight Server (vineyard + groot) + enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; -option java_package = "com.alibaba.graphscope.proto"; -option java_multiple_files = true; + // todo: consider to add an UNKNOWN error code in the common error code? + UNKNOWN_ERROR = 14; + + // try to get a property in ValueType1, but the it's real type is ValueType2 and it's not ValueType1 and + // compatible to ValueType1 + VALUE_TYPE_MISMATCH = 100; + // get lock failed + LOCK_FAILED = 101; + // too many data of old versions in graph store, maybe something error with garbage collection + // usuage trigger when modifing the schema + TOO_MANY_VERSIONS = 102; + // some fatal bug in graph store + // GraphStoreBug = 103; + BACKUP_FAILED = 103; + // user's operation is invalid, like: create a type that already exists + INVALID_OPERATION = 104; + // when try to insert data, the type exists in storage but isn't visible at that moment + DATA_NOT_EXISTS = 105; + // error in external storage like rocksdb + EXTERNAL_STORAGE_ERROR = 106; + // decode property from bytes failed, maybe caused by corrupted binary data in storage, + // or by using the wrong decoder, + // DecodeError = 107; + INVALID_DATA = 107; + // operations or features is not supported, e.g. delete from a secondary instance + // NotSupported = 108; + + // PropertyAlreadyExist = 109; + TYPE_NOT_FOUND = 108; + PROPERTY_NOT_FOUND = 109; + // TypeAlreadyExist = 111; + // RelationShipAlreadyExist = 113; + // RelationShipNotExist = 114; + + // client-frontend related + REALTIME_WRITE_FAILED = 110; + SERVICE_NOT_READY = 111; + QUEUE_REJECT = 112; + QUERY_FAILED = 113; + + ILLEGAL_SCHEMA = 115; + // e.g. delete property + // UnSupportedMetaOperation = 115; + // e.g. list, map, set + // DataTypeNotValid = 109; + INVALID_DATA_TYPE = 116; + + INTERNAL = 117; + + GAIA_INTERNAL_ERROR = 118; + + DDL_ERROR = 119; + + // pegasus server and runtime related errors, with code 2xx + // 1. submit job error + JOB_SUBMIT_BUILD_JOB_UNSUPPORTED = 200; + JOB_SUBMIT_BUILD_JOB_INTERNAL_ERROR = 201; + JOB_SUBMIT_BUILD_JOB_SERVER_ERROR = 202; + JOB_SUBMIT_BUILD_JOB_USER_ERROR = 203; + JOB_SUBMIT_SPAWN_JOB_ERROR = 204; -// component-04: GIE Insight Server (vineyard + groot) -enum Code { - // Not an error; returned on success. - OK = 0; - CANCELLED = 1; - // Arguments of operations is invalid / in a wrong format. - INVALID_ARGUMENT = 2; - // Timeout, used when an operation fail to return result in an specific time. - TIMEOUT = 3; - // Required resources cannot be found. - NOT_FOUND = 4; - // File or resources already existed. - ALREADY_EXISTS = 5; - RESOURCE_EXHAUSTED = 6; - // Functionality not implemented yet - UNIMPLEMENTED = 7; - // Client doesn't have the permission. - PERMISSION_DENIED = 8; - // Compile graph frame or app frame failed. - COMPILATION_FAILURE = 9; - PORT_IN_USE = 10; - UNSUPPORTED_OPERATION = 11; - ILLEGAL_STATE = 12; - // Network is unreachable - NETWORK_FAILURE = 13; - - // try to get a property in ValueType1, but the it's real type is ValueType2 and it's not ValueType1 and - // compatible to ValueType1 - // ValueTypeMismatch = 100; - VALUE_TYPE_MISMATCH = 100; - // get lock failed - // LockFailed = 101; - LOCK_FAILED = 101; - // too many data of old versions in graph store, maybe something error with garbage collection - // usuage trigger when modifing the schema - // TooManyVersions = 102; - TOO_MANY_VERSIONS = 102; - // some fatal bug in graph store - // GraphStoreBug = 103; - BACKUP_FAILED = 103; - // user's operation is invalid, like: create a type that already exists - // InvalidOperation = 104; - INVALID_OPERATION = 104; - // when try to insert data, the type exists in storage but isn't visible at that moment - // DataNotExists = 105; - DATA_NOT_EXISTS = 105; - // error in external storage like rocksdb - // ExternalStorageError = 106; - EXTERNAL_STORAGE_ERROR = 106; - // decode property from bytes failed, maybe caused by corrupted binary data in storage, - // or by using the wrong decoder, - // DecodeError = 107; - INVALID_DATA = 107; - // operations or features is not supported, e.g. delete from a secondary instance - // NotSupported = 108; - - // PropertyAlreadyExist = 109; - TYPE_NOT_FOUND = 108; - PROPERTY_NOT_FOUND = 109; - // TypeAlreadyExist = 111; - // RelationShipAlreadyExist = 113; - // RelationShipNotExist = 114; - - // client-frontend related - // RealtimeWriteFailed = 110; - REALTIME_WRITE_FAILED = 110; - // ServiceNotReady = 111; - SERVICE_NOT_READY = 111; - // QueueReject = 112; - QUEUE_REJECT = 112; - - // query related - // QueryFailed = 113; - QUERY_FAILED = 113; - // QueryCancelled = 114; - // QUERY_CANCELLED = 114; - - // IllegalSchema = 115; - ILLEGAL_SCHEMA = 115; - - // e.g. delete property - // UnSupportedMetaOperation = 115; - // e.g. list, map, set - // DataTypeNotValid = 109; - INVALID_DATA_TYPE = 116; - - INTERNAL = 117; - - GAIA_INTERNAL_ERROR = 118; - - DDL_ERROR = 119; - -} + // 2. execute job error + JOB_EXECUTE_WOULD_BLOCK = 211; + JOB_EXECUTE_INTERRUPTED = 212; + JOB_EXECUTE_IO_ERROR = 213; + JOB_EXECUTE_ILLEAGAL_SCOPE_INPUT = 214; + JOB_EXECUTE_CANCELLED = 215; + JOB_EXECUTE_OTHERS = 216; + } + \ No newline at end of file diff --git a/python/graphscope/gsctl/commands/dev.py b/python/graphscope/gsctl/commands/dev.py index 6a5d7968ec97..8138ebb14b62 100644 --- a/python/graphscope/gsctl/commands/dev.py +++ b/python/graphscope/gsctl/commands/dev.py @@ -21,6 +21,7 @@ import io import os import subprocess +import sys import click from packaging import version @@ -62,6 +63,8 @@ def run_shell_cmd(cmd, workingdir): ) for line in io.TextIOWrapper(proc.stdout, encoding="utf-8"): print(line.rstrip()) + proc.wait() + sys.exit(proc.returncode) @click.group() diff --git a/python/graphscope/gsctl/scripts/install_deps_command.sh b/python/graphscope/gsctl/scripts/install_deps_command.sh index 0925aceb458f..a835b690f8cd 100644 --- a/python/graphscope/gsctl/scripts/install_deps_command.sh +++ b/python/graphscope/gsctl/scripts/install_deps_command.sh @@ -234,14 +234,18 @@ install_basic_packages_universal() { elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then if [[ "${OS_VERSION}" -eq "7" ]]; then ${SUDO} yum install -y ${BASIC_PACKAGES_CENTOS_7[*]} + # change the source for centos-release-scl-rh + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* + ${SUDO} sed -i 's|# baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*scl* ${SUDO} yum install -y ${ADDITIONAL_PACKAGES_CENTOS_7[*]} else if [[ "${OS_PLATFORM}" == *"Aliyun"* ]]; then ${SUDO} yum install -y 'dnf-command(config-manager)' ${SUDO} dnf install -y epel-release --allowerasing else - sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + ${SUDO} sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* ${SUDO} yum install -y 'dnf-command(config-manager)' ${SUDO} dnf install -y epel-release ${SUDO} dnf config-manager --set-enabled powertools @@ -324,7 +328,12 @@ install_dependencies_analytical_universal() { } install_interactive_deps() { - install_hiactor "${install_prefix}" + # seastar can not be built on macos and centos7 + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + install_hiactor "${install_prefix}" + else + warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." + fi } write_env_config() { @@ -414,12 +423,7 @@ install_deps_for_dev() { install_rust_universal install_cppkafka "${deps_prefix}" "${install_prefix}" # install dependencies for flex interactive - # can not install on macos since seastar can not be built on macos - if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - warning "Skip installing dependencies for flex interactive on macOS." - else - install_interactive_deps - fi + install_interactive_deps fi write_env_config diff --git a/python/jupyter/graphscope/.eslintignore b/python/jupyter/graphscope/.eslintignore deleted file mode 100644 index 5c99ba78a7a2..000000000000 --- a/python/jupyter/graphscope/.eslintignore +++ /dev/null @@ -1,5 +0,0 @@ -node_modules -dist -coverage -**/*.d.ts -tests diff --git a/python/jupyter/graphscope/.eslintrc.js b/python/jupyter/graphscope/.eslintrc.js deleted file mode 100644 index 3cb48f2c6be1..000000000000 --- a/python/jupyter/graphscope/.eslintrc.js +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -module.exports = { - extends: [ - 'eslint:recommended', - 'plugin:@typescript-eslint/eslint-recommended', - 'plugin:@typescript-eslint/recommended', - 'plugin:prettier/recommended', - ], - parser: '@typescript-eslint/parser', - parserOptions: { - project: 'tsconfig.json', - sourceType: 'module' - }, - plugins: ['@typescript-eslint'], - rules: { - '@typescript-eslint/ban-ts-ignore': 'off', //stackoverflow.com/questions/59729654/how-ignore-typescript-errors-with-ts-ignore - '@typescript-eslint/camelcase': 'off', - '@typescript-eslint/explicit-function-return-type': 'off', - '@typescript-eslint/interface-name-prefix': [ - 'error', - { prefixWithI: 'always' }, - ], - '@typescript-eslint/no-unused-vars': ['warn', { args: 'none' }], - '@typescript-eslint/no-explicit-any': 'off', - '@typescript-eslint/no-namespace': 'off', - '@typescript-eslint/no-this-alias': [ - 'error', - { - allowedNames: ['self'], // Allow `const self = this` - }, - ], - '@typescript-eslint/no-use-before-define': 'off', - '@typescript-eslint/quotes': [ - 'error', - 'single', - { avoidEscape: true, allowTemplateLiterals: false } - ], - curly: ['error', 'all'], - eqeqeq: 'error', - 'prefer-arrow-callback': 'error' - } - }; - diff --git a/python/jupyter/graphscope/.gitignore b/python/jupyter/graphscope/.gitignore deleted file mode 100644 index a1f97911cac4..000000000000 --- a/python/jupyter/graphscope/.gitignore +++ /dev/null @@ -1,161 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask instance folder -instance/ - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ -docs/source/_static/embed-bundle.js -docs/source/_static/embed-bundle.js.map - -# PyBuilder -target/ - -# IPython Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# dotenv -.env - -# virtualenv -venv/ -ENV/ - -# Spyder project settings -.spyderproject - -# Rope project settings -.ropeproject - -# ========================= -# Operating System Files -# ========================= - -# OSX -# ========================= - -.DS_Store -.AppleDouble -.LSOverride - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -# Windows -# ========================= - -# Windows image file caches -Thumbs.db -ehthumbs.db - -# Folder config file -Desktop.ini - -# Recycle Bin used on file shares -$RECYCLE.BIN/ - -# Windows Installer files -*.cab -*.msi -*.msm -*.msp - -# Windows shortcuts -*.lnk - - -# NPM -# ---- - -**/node_modules/ -graphscope-jupyter/labextension/*.tgz -graphscope-jupyter/nbextension/static/ -graphscope_jupyter/labextension/*.tgz -graphscope_jupyter/nbextension/static/ - -# Coverage data -# ------------- -**/coverage/ - -# Packed lab extensions -graphscope-jupyter/labextension -graphscope_jupyter/labextension - -MANIFEST diff --git a/python/jupyter/graphscope/.npmignore b/python/jupyter/graphscope/.npmignore deleted file mode 100644 index f8ec1d1968b2..000000000000 --- a/python/jupyter/graphscope/.npmignore +++ /dev/null @@ -1,7 +0,0 @@ -.DS_Store -node_modules/ -tests/ -.jshintrc -# Ignore any build output from python: -dist/*.tar.gz -dist/*.wheel diff --git a/python/jupyter/graphscope/.prettierignore b/python/jupyter/graphscope/.prettierignore deleted file mode 100644 index 8103ecd203bc..000000000000 --- a/python/jupyter/graphscope/.prettierignore +++ /dev/null @@ -1,4 +0,0 @@ -node_modules -**/node_modules -**/lib -**/package.json diff --git a/python/jupyter/graphscope/.prettierrc b/python/jupyter/graphscope/.prettierrc deleted file mode 100644 index 544138be4565..000000000000 --- a/python/jupyter/graphscope/.prettierrc +++ /dev/null @@ -1,3 +0,0 @@ -{ - "singleQuote": true -} diff --git a/python/jupyter/graphscope/LICENSE b/python/jupyter/graphscope/LICENSE deleted file mode 100644 index 4c9ad9806822..000000000000 --- a/python/jupyter/graphscope/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/python/jupyter/graphscope/MANIFEST.in b/python/jupyter/graphscope/MANIFEST.in deleted file mode 100644 index fc18d77ae50c..000000000000 --- a/python/jupyter/graphscope/MANIFEST.in +++ /dev/null @@ -1,40 +0,0 @@ -include LICENSE -include README.md - -include setupbase.py -include pytest.ini -include .coverage.rc - -include tsconfig.json -include package.json -include webpack.config.js -include graphscope-jupyter/labextension/*.tgz - -# Documentation -graft docs -exclude docs/\#* -prune docs/build -prune docs/gh-pages -prune docs/dist - -# Examples -graft examples - -# Tests -graft tests -prune tests/build - -# JavaScript files -graft graphscope-jupyter/nbextension -graft src -graft css -prune **/node_modules -prune coverage -prune lib - -# Patterns to exclude from any directory -global-exclude *~ -global-exclude *.pyc -global-exclude *.pyo -global-exclude .git -global-exclude .ipynb_checkpoints diff --git a/python/jupyter/graphscope/README.md b/python/jupyter/graphscope/README.md index 15909b9d421b..8654ad107543 100644 --- a/python/jupyter/graphscope/README.md +++ b/python/jupyter/graphscope/README.md @@ -1,140 +1,2 @@ -# graphscope-jupyter - -The project structure refers to **(ipycytoscape)[https://github.com/QuantStack/ipycytoscape/tree/1.1.0]**. - -A widget enabling interactive graph visualization with [Graphin](https://github.com/antvis/Graphin) in JupyterLab and the Jupyter notebook. - -![graphin screencast](https://gw.alipayobjects.com/mdn/rms_f8c6a0/afts/img/A*EJvtT7KcywAAAAAAAAAAAAAAARQnAQ) - -## Installation - -Note that graphscope-jupyter requires jupyterlab 2.x (e.g., `2.3.0a0`) and is known unusable with jupyterlab 3.x. - -With `mamba`: - -``` -mamba install -c conda-forge graphscope-jupyter -``` - -With `conda`: - -``` -conda install -c conda-forge graphscope-jupyter -``` - -With `pip`: - -```bash -pip install graphscope-jupyter -``` - -#### For jupyterlab users: - -If you are using JupyterLab 1.x or 2.x then you will also need to install `nodejs` and the `jupyterlab-manager` extension. You can do this like so: - -```bash -# installing nodejs -conda install -c conda-forge nodejs - - -# install jupyterlab-manager extension -jupyter labextension install @jupyter-widgets/jupyterlab-manager@2 - -# if you have previously installed the manager you still to run jupyter lab build -jupyter lab build -``` - -### For Jupyter Notebook 5.2 and earlier - -You may also need to manually enable the nbextension: - -```bash -jupyter nbextension enable --py [--sys-prefix|--user|--system] graphscope-jupyter -``` - -## For a development installation: - -**(requires npm)** - -While not required, we recommend creating a conda environment to work in: - -```bash -conda create -n graphscope -c conda-forge jupyterlab nodejs networkx -conda activate graphscope - -# clone repo -git clone https://github.com/alibaba/GraphScope.git -cd GraphScope/python/jupyter/graphscope - -# Install python package for development, runs npm install and npm run build -pip install -e . -``` - -When developing graphscope-jupyter, you need to manually enable the extension with the -notebook / lab frontend. For lab, this is done by the command: - -``` -# install this extension -jupyter labextension install . -``` - -For classic notebook, you can run: - -``` -jupyter nbextension install --sys-prefix --symlink --overwrite --py graphscope-jupyter -jupyter nbextension enable --sys-prefix --py graphscope-jupyter -``` - -Note that the `--symlink` flag doesn't work on Windows, so you will here have to run -the `install` command every time that you rebuild your extension. For certain installations -you might also need another flag instead of `--sys-prefix`, but we won't cover the meaning -of those flags here. - -You need to install and build `npm` packages: - -``` -npm install && npm run build -``` - -Every time you change your typescript code it's necessary to build it again: - -``` -npm run build -``` - -### How to see your changes - -#### TypeScript: - -To continuously monitor the project for changes and automatically trigger a rebuild, start Jupyter in watch mode: - -```bash -jupyter lab --watch -``` - -And in a separate session, begin watching the source directory for changes: - -```bash -npm run watch -``` - -#### Python: - -If you make a change to the python code then you need to restart the notebook kernel to have it take effect. - -### How to run tests locally - -Install necessary dependencies with pip: - -``` -cd GraphScope/python/jupyter/graphscope -pip install -e . -``` - -## License - -We use a shared copyright model that enables all contributors to maintain the -copyright on their contributions. - -This software is licensed under the Apache License 2.0. See the -[LICENSE](LICENSE) file for details. +> [!NOTE] +> The Jupyter Notebook extension has been deprecated. We are currently developing a new visualization tool in Python, which we plan to release by the end of 2024. If you still need access to the legacy extension, which was previously available here, please refer to this [archived repository](https://github.com/GraphScope/jupyter-legacy). \ No newline at end of file diff --git a/python/jupyter/graphscope/css/index.css b/python/jupyter/graphscope/css/index.css deleted file mode 100644 index 6399d6e8d9f7..000000000000 --- a/python/jupyter/graphscope/css/index.css +++ /dev/null @@ -1,10 +0,0 @@ -.custom-widget { - background-color: white; - height: 400px; - width: 100%; - resize: both; -} - -.custom-widget:hover { - border: 1px dashed #ccc; -} diff --git a/python/jupyter/graphscope/dodo.py b/python/jupyter/graphscope/dodo.py deleted file mode 100644 index 413d97e9180f..000000000000 --- a/python/jupyter/graphscope/dodo.py +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# this uses https://pydoit.org/ to run tasks/chores -# pip install doit -# $ doit - -import re - -from graphscope_jupyter._version import __version__ as version - - -def task_mybinder(): - """Make the mybinder files up to date""" - - def action(targets): - for filename in targets: - with open(filename) as f: - content = f.read() - content = re.sub( - "graphin(?P[^0-9]*)([0-9\.].*)", # noqa: W605 - rf"graphin\g{version}", - content, - ) - with open(filename, "w") as f: - f.write(content) - print(f"{filename} updated") - - return { - "actions": [action], - "targets": ["environment.yml", "postBuild"], - "file_dep": ["graphscope_jupyter/_version.py"], - } diff --git a/python/jupyter/graphscope/environment.yml b/python/jupyter/graphscope/environment.yml deleted file mode 100644 index d15db7cf7dae..000000000000 --- a/python/jupyter/graphscope/environment.yml +++ /dev/null @@ -1,11 +0,0 @@ -name: graphscope-jupyter -channels: - - conda-forge -dependencies: - - graphscope-jupyter>=0.1.0 - - jupyterlab=2 - - matplotlib-base - - networkx - - pandas - - traitlets - - spectate diff --git a/python/jupyter/graphscope/graphscope-jupyter.json b/python/jupyter/graphscope/graphscope-jupyter.json deleted file mode 100644 index cf9327a99a5d..000000000000 --- a/python/jupyter/graphscope/graphscope-jupyter.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "load_extensions": { - "graphscope-jupyter/extension": true - } -} diff --git a/python/jupyter/graphscope/graphscope_jupyter/__init__.py b/python/jupyter/graphscope/graphscope_jupyter/__init__.py deleted file mode 100644 index 331ee171074b..000000000000 --- a/python/jupyter/graphscope/graphscope_jupyter/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from ._version import __version__ -from ._version import version_info -from .graphin import * -from .nbextension import _jupyter_nbextension_paths diff --git a/python/jupyter/graphscope/graphscope_jupyter/_frontend.py b/python/jupyter/graphscope/graphscope_jupyter/_frontend.py deleted file mode 100644 index 397168d1a55c..000000000000 --- a/python/jupyter/graphscope/graphscope_jupyter/_frontend.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Information about the frontend package of the widgets. -""" - -from ._version import __version__ - -module_name = "@graphscope/graphscope-jupyter" -module_version = "^%s" % __version__ diff --git a/python/jupyter/graphscope/graphscope_jupyter/_version.py b/python/jupyter/graphscope/graphscope_jupyter/_version.py deleted file mode 100644 index f7ebffc1ec21..000000000000 --- a/python/jupyter/graphscope/graphscope_jupyter/_version.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020-2021 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from graphscope.version import __version__ -from graphscope.version import __version_tuple__ - -version_info = __version_tuple__ diff --git a/python/jupyter/graphscope/graphscope_jupyter/graphin.py b/python/jupyter/graphscope/graphscope_jupyter/graphin.py deleted file mode 100644 index fc1fc1b911d8..000000000000 --- a/python/jupyter/graphscope/graphscope_jupyter/graphin.py +++ /dev/null @@ -1,407 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import sys - -import ipywidgets as widgets -from spectate import mvc -from traitlets import TraitType -from traitlets import Unicode - -from ._frontend import module_name -from ._frontend import module_version - - -class Mutable(TraitType): - """A base class for mutable traits using Spectate""" - - _model_type = None - _event_type = "change" - - def instance_init(self, obj): - default = self._model_type() - - @mvc.view(default) - def callback(default, events): - change = dict( - new=getattr(obj, self.name), - name=self.name, - type=self._event_type, - ) - obj.notify_change(change) - - setattr(obj, self.name, default) - - -class MutableDict(Mutable): - """A mutable dictionary trait""" - - _model_type = mvc.Dict - - -@widgets.register -class GraphModel(widgets.DOMWidget): - """Graph Widget""" - - # Name of the widget model class in front-end - _model_name = Unicode("GraphModel").tag(sync=True) - - # Name of the front-end module containing widget model - _model_module = Unicode(module_name).tag(sync=True) - - # Version of the front-end module containing widget model - _model_module_version = Unicode(module_version).tag(sync=True) - - # Name of the widget view class in front-end - _view_name = Unicode("GraphView").tag(sync=True) - - # Name of the front-end module containing widget view - _view_module = Unicode(module_name).tag(sync=True) - - # Version of the front-end module containing widget view - _view_module_version = Unicode(module_version).tag(sync=True) - - # Widget specific property. - # Widget properties are defined as traitlets. Any property tagged with `sync=True` - # is automatically synced to the frontend *any* time it changes in Python. - # It is synced back to Python from the frontend *any* time the model is touched. - # data: { nodes: [], edges: [] } - - value_dict = {} - value = Unicode("").tag(sync=True) - - _interactive_query = None - - _nodes_id_map = {} - _nodes_id_dict = {} - _edges_id_dict = {} - - _default_data = { - "nodes": [ - { - "id": "1", - "label": "person", - "nodeType": "person", - }, - { - "id": "2", - "label": "person", - "nodeType": "person", - }, - { - "id": "3", - "label": "person", - "nodeType": "person", - }, - ], - "edges": [ - { - "label": "knows", - "source": "1", - "target": "2", - }, - { - "label": "knows", - "source": "2", - "target": "3", - }, - { - "label": "knows", - "source": "3", - "target": "1", - }, - ], - } - - def addGraphFromData(self, data=None): - if data is None: - data = self._default_data - nodeList = [] - edgeList = [] - - def _addNodes(nodes): - for node in nodes: - current = {} - current["id"] = str(node["id"]) - current["label"] = node["label"] - current["parentId"] = "" - current["level"] = 0 - current["degree"] = 1 # need to update - current["count"] = 0 - current["nodeType"] = node["nodeType"] - current["properties"] = {} - nodeList.append(current) - - def _addEdges(list_edge): - for e in list_edge: - edge = {} - edge["label"] = e["label"] - edge["source"] = str(e["source"]) - edge["target"] = str(e["target"]) - edge["count"] = 0 - # edge["edgeType"] = e['edgeType'] - edge["properties"] = {} - edgeList.append(edge) - - _addNodes(data["nodes"]) - _addEdges(data["edges"]) - - data_dict = {} - data_dict["graphVisId"] = "0" - data_dict["nodes"] = nodeList - data_dict["edges"] = edgeList - - data_str = json.dumps(data_dict) - self.value = data_str - - def _gremlin(self, query=""): - return self._interactive_query.execute(query).all().result() - - def _process_vertices_1_hop(self, vertices): - nodes = [] - edges = [] - - def _process_node(list_id, list_val, list_prop): - for i, item in enumerate(list_id): - vid = str(item.id) - if vid in self._nodes_id_dict: - continue - # - node = {} - node["id"] = vid - node["oid"] = str(list_val[i]) - node["parentId"] = "" - node["label"] = str(item.label) - node["level"] = 0 - node["degree"] = 1 # need to update - node["count"] = 0 - node["nodeType"] = str(item.label) - node["properties"] = list_prop[i] - self._nodes_id_dict[vid] = True - self._nodes_id_map[vid] = str(list_val[i]) - nodes.append(node) - - def _process_edge(list_edge): - for e in list_edge: - edge = {} - edge["id"] = str(e.id) - # - if edge["id"] in self._edges_id_dict: - continue - # - edge["label"] = e.label - edge["source"] = str(e.outV.id) - edge["target"] = str(e.inV.id) - # edge["source"] = self._nodes_id_map[str(e.outV.id)] - # edge["target"] = self._nodes_id_map[str(e.inV.id)] - edge["count"] = 0 - edge["edgeType"] = e.label - edge["properties"] = {} - self._edges_id_dict[edge["id"]] = True - edges.append(edge) - - for vert in vertices: - vert_str = str(vert) - # node - list_id = self._gremlin("g.V().has('id'," + vert_str + ")") - list_id_val = self._gremlin("g.V().has('id'," + vert_str + ").values('id')") - list_id_prop = self._gremlin("g.V().has('id'," + vert_str + ").valueMap()") - _process_node(list_id, list_id_val, list_id_prop) - # - list_id_inV = self._gremlin( - "g.V().has('id'," + vert_str + ").outE().inV().order().by('id',incr)" - ) - list_id_inV_val = self._gremlin( - "g.V().has('id'," - + vert_str - + ").outE().inV().order().by('id',incr).values('id')" - ) - list_id_inV_prop = self._gremlin( - "g.V().has('id'," - + vert_str - + ").outE().inV().order().by('id',incr).valueMap()" - ) - _process_node(list_id_inV, list_id_inV_val, list_id_inV_prop) - # - list_id_outV = self._gremlin( - "g.V().has('id'," + vert_str + ").inE().outV().order().by('id',incr)" - ) - list_id_outV_val = self._gremlin( - "g.V().has('id'," - + vert_str - + ").inE().outV().order().by('id',incr).values('id')" - ) - list_id_outV_prop = self._gremlin( - "g.V().has('id'," - + vert_str - + ").inE().outV().order().by('id',incr).valueMap()" - ) - _process_node(list_id_outV, list_id_outV_val, list_id_outV_prop) - # edge - list_edge = self._gremlin( - "g.V().has('id'," + vert_str + ").union(outE(), inE())" - ) - _process_edge(list_edge) - - data_dict = {} - data_dict["graphVisId"] = "0" - data_dict["nodes"] = nodes - data_dict["edges"] = edges - - return data_dict - - def queryGraphData(self, vertices, hop, interactive_query=None): - """ - Set JSON value to `data` after query gremlin server. - - Args: - vertices (list): Vertex Id list. - hop (int): Number of top. Default to 1. - interactive_query (:class:`InteractiveQuery`): Gremlin server instance. - - Returns: None - """ - if not isinstance(vertices, (list, tuple, range)): - vertices = [vertices] - hop = int(hop) - - if interactive_query is not None: - self._interactive_query = interactive_query - - if self._interactive_query is None: - raise ValueError( - "Failed to obtain interactive_query, unable to query data and draw graph." - ) - - if hop == 1: - self.value_dict = self._process_vertices_1_hop(vertices) - self.value = json.dumps(self.value_dict) - else: - raise NotImplementedError - - def queryNeighbor(self, ins, params, buffers): - """ - Args: - ins: Listening required. - params (dict): Contains "degree" and "nodeId" of node. - buffers (bool): Listening required. - """ - if "nodeId" in params and "degree" in params: - vid = str(params["nodeId"]) - # convert to original id - oid = self._nodes_id_map[vid] - hop = int(params["degree"]) - - if hop == 1: - new_value_dict = self._process_vertices_1_hop([oid]) - self.value_dict["nodes"].extend(new_value_dict["nodes"]) - self.value_dict["edges"].extend(new_value_dict["edges"]) - self.value = json.dumps(self.value_dict) - else: - raise NotImplementedError - - -def draw_graphscope_graph(graph, vertices, hop=1): - """Visualize the graph data in the result cell when the draw functions are invoked - - Args: - vertices (list): selected vertices. - hop (int): draw induced subgraph with hop extension. Defaults to 1. - - Returns: - A GraphModel. - """ - graph._ensure_loaded() - interactive_query = graph._session.gremlin(graph) - - gm = GraphModel() - - # for debugging - # gm.addGraphFromData() - - gm.queryGraphData(vertices, hop, interactive_query) - # listen on the 1~2 hops operation of node - gm.on_msg(gm.queryNeighbor) - - return gm - - -def repr_graphscope_graph(graph, *args, **kwargs): - from ipywidgets.widgets.widget import Widget - - if "_ipython_display_" in Widget.__dict__: - return draw_graphscope_graph(graph, vertices=range(1, 100))._ipython_display_( - *args, **kwargs - ) - return draw_graphscope_graph(graph, vertices=range(1, 100))._repr_mimebundle_( - *args, **kwargs - ) - - -def in_ipython(): - try: - get_ipython().__class__.__name__ - return True - except NameError: - return False - - -def in_notebook(): - try: - shell = get_ipython().__class__.__name__ - if shell == "ZMQInteractiveShell": - return True # Jupyter notebook or qtconsole - if shell == "TerminalInteractiveShell": - return False # Terminal running IPython - return False # Other type (?) - except NameError: - return False # Probably standard Python interpreter - - -def __graphin_for_graphscope(graphscope): - if in_notebook(): - graph_type = getattr(graphscope, "Graph") - setattr(graph_type, "draw", draw_graphscope_graph) - from ipywidgets.widgets.widget import Widget - - if "_ipython_display_" in Widget.__dict__: - setattr(graph_type, "_ipython_display_", repr_graphscope_graph) - else: - setattr(graph_type, "_repr_mimebundle_", repr_graphscope_graph) - - -def __register_graphin_for_graphscope(): - # if graphscope already loaded - if "graphscope" in sys.modules: - __graphin_for_graphscope(sys.modules["graphscope"]) # noqa: F821 - - hookpoint = get_ipython().user_ns # noqa: F821 - - # added to graphscope extension lists - if "__graphscope_extensions__" not in hookpoint: - hookpoint["__graphscope_extensions__"] = [] - hookpoint["__graphscope_extensions__"].append( - __graphin_for_graphscope # noqa: F821 - ) - - -if in_ipython(): - __register_graphin_for_graphscope() -del __graphin_for_graphscope -del __register_graphin_for_graphscope diff --git a/python/jupyter/graphscope/graphscope_jupyter/nbextension/__init__.py b/python/jupyter/graphscope/graphscope_jupyter/nbextension/__init__.py deleted file mode 100644 index 02369f405d20..000000000000 --- a/python/jupyter/graphscope/graphscope_jupyter/nbextension/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - - -def _jupyter_nbextension_paths(): - return [ - { - "section": "notebook", - "src": "nbextension/static", - "dest": "graphscope-jupyter", - "require": "graphscope-jupyter/extension", - } - ] diff --git a/python/jupyter/graphscope/package.json b/python/jupyter/graphscope/package.json deleted file mode 100644 index c58383c6a652..000000000000 --- a/python/jupyter/graphscope/package.json +++ /dev/null @@ -1,118 +0,0 @@ -{ - "name": "@graphscope/graphscope-jupyter", - "version": "0.4.2", - "description": "A GraphScope Jupyter Extension, including Graphin Widget, etc.", - "keywords": [ - "jupyter", - "jupyter notebook", - "notebook", - "jupyterlab", - "jupyterlab-extension", - "widgets", - "graphin", - "G6" - ], - "files": [ - "{dist,lib}/**/*.{js,ts,map}", - "css/*.css", - "LICENSE" - ], - "homepage": "https://github.com/antvis/G6", - "bugs": { - "url": "https://github.com/antvis/G6/issues" - }, - "license": "BSD-3-Clause", - "author": "baizn <576375879@qq.com>", - "main": "src/index.ts", - "types": "./lib/index.d.ts", - "repository": { - "type": "git", - "url": "git+https://github.com/antvis/G6.git" - }, - "scripts": { - "build": "npm run build:lib && npm run build:nbextension", - "build:labextension": "npm run clean:labextension && mkdirp graphscope-jupyter/labextension && cd graphscope-jupyter/labextension && npm pack ../..", - "build:lib": "tsc", - "build:nbextension": "NODE_OPTIONS=--max_old_space_size=2048 webpack -p ", - "build:all": "npm run build:labextension && npm run build:nbextension", - "clean": "npm run clean:lib && npm run clean:nbextension", - "clean:lib": "rimraf lib", - "clean:labextension": "rimraf graphscope-jupyter/labextension", - "clean:nbextension": "rimraf graphscope-jupyter/nbextension/static/index.js", - "lint": "eslint . --ext .ts --fix", - "lint-check": "eslint . --ext .ts", - "prepack": "npm run build:lib", - "test": "npm run test:firefox", - "test:chrome": "karma start --browsers=Chrome tests/karma.conf.js", - "test:debug": "karma start --browsers=Chrome --singleRun=false --debug=true tests/karma.conf.js", - "test:firefox": "karma start --browsers=Firefox tests/karma.conf.js", - "test:ie": "karma start --browsers=IE tests/karma.conf.js", - "watch": "npm-run-all -p watch:*", - "watch:lib": "tsc -w", - "watch:nbextension": "webpack --watch" - }, - "dependencies": { - "@antv/graphin-graphscope": "^1.0.3", - "@jupyter-widgets/base": "^1.1.10 || ^2 || ^3", - "react": "^16.9.0", - "react-dom": "^16.9.0" - }, - "devDependencies": { - "@phosphor/application": "^1.6.0", - "@phosphor/widgets": "^1.6.0", - "@types/expect.js": "^0.3.29", - "@types/mocha": "^5.2.5", - "@types/node": "^10.11.6", - "@types/react": "^16.9.0", - "@types/react-dom": "^16.9.0", - "@types/webpack-env": "^1.13.6", - "@typescript-eslint/eslint-plugin": "^2.31.0", - "@typescript-eslint/parser": "^2.31.0", - "css-loader": "^3.2.0", - "eslint": "^6.8.0", - "eslint-config-prettier": "^6.11.0", - "eslint-plugin-prettier": "^3.1.3", - "expect.js": "^0.3.1", - "fs-extra": "^7.0.0", - "husky": "^4.2.5", - "karma": "^6.3.16", - "karma-chrome-launcher": "^2.2.0", - "karma-firefox-launcher": "^1.1.0", - "karma-ie-launcher": "^1.0.0", - "karma-mocha": "^1.3.0", - "karma-mocha-reporter": "^2.2.5", - "karma-typescript": "^5.5.1", - "lint-staged": "^10.2.2", - "mkdirp": "^0.5.1", - "mocha": "^5.2.0", - "npm-run-all": "^4.1.3", - "prettier": "^2.0.5", - "rimraf": "^2.6.2", - "source-map-loader": "^0.2.4", - "style-loader": "^1.0.0", - "ts-loader": "^5.2.1", - "typescript": "^3.7.1", - "webpack": "^4.20.2", - "webpack-cli": "^3.1.2" - }, - "jupyterlab": { - "extension": "lib/plugin" - }, - "husky": { - "hooks": { - "pre-commit": "lint-staged" - } - }, - "lint-staged": { - "*.ts": [ - "eslint . --ext .ts --fix" - ] - }, - "prettier": { - "singleQuote": true - }, - "directories": { - "doc": "docs", - "lib": "lib" - } -} diff --git a/python/jupyter/graphscope/postBuild b/python/jupyter/graphscope/postBuild deleted file mode 100644 index 4369cd18e091..000000000000 --- a/python/jupyter/graphscope/postBuild +++ /dev/null @@ -1 +0,0 @@ -jupyter labextension install @jupyter-widgets/jupyterlab-manager graphscope-jupyter@0.0.1 diff --git a/python/jupyter/graphscope/setup.cfg b/python/jupyter/graphscope/setup.cfg deleted file mode 100644 index f0e7be775530..000000000000 --- a/python/jupyter/graphscope/setup.cfg +++ /dev/null @@ -1,6 +0,0 @@ -[bdist_wheel] -universal=1 - -[metadata] -description-file = README.md -license_file = LICENSE diff --git a/python/jupyter/graphscope/setup.py b/python/jupyter/graphscope/setup.py deleted file mode 100644 index 9337c7ca3c05..000000000000 --- a/python/jupyter/graphscope/setup.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -from __future__ import print_function - -from glob import glob -from os import path -from os.path import join as pjoin - -from setupbase import HERE -from setupbase import combine_commands -from setupbase import create_cmdclass -from setupbase import ensure_targets -from setupbase import find_packages -from setupbase import get_version -from setupbase import install_npm -from setuptools import setup - -from graphscope import __version__ - -# The name of the project -name = "graphscope-jupyter" -package_name = "graphscope_jupyter" - -# Get our version -version = __version__ - -nb_path = pjoin(HERE, package_name, "nbextension", "static") -lab_path = pjoin(HERE, package_name, "labextension") - -# Representative files that should exist after a successful build -jstargets = [ - pjoin(nb_path, "index.js"), - pjoin(HERE, "lib", "plugin.js"), -] - -package_data_spec = {package_name: ["nbextension/static/*.*js*", "labextension/*.tgz"]} - -data_files_spec = [ - ("share/jupyter/nbextensions/graphscope-jupyter", nb_path, "*.js*"), - ("share/jupyter/lab/extensions", lab_path, "*.tgz"), - ("etc/jupyter/nbconfig/notebook.d", HERE, "graphscope-jupyter.json"), -] - - -cmdclass = create_cmdclass( - "jsdeps", - package_data_spec=package_data_spec, - data_files_spec=data_files_spec, # noqa: E501 -) -cmdclass["jsdeps"] = combine_commands( - install_npm(HERE, build_cmd="build:all"), - ensure_targets(jstargets), -) - -# Read the contents of the README file on Pypi -this_directory = path.abspath(path.dirname(__file__)) -with open(pjoin(this_directory, "README.md"), encoding="utf-8") as f: - long_description = f.read() - -setup_args = dict( - name=name, - description="Python implementation of the graph visualization tool Graphin.", # noqa: E501 - long_description=long_description, - long_description_content_type="text/markdown", - version=version, - scripts=glob(pjoin("scripts", "*")), - cmdclass=cmdclass, - packages=find_packages(), - author="Alibaba Damo Academy", - author_email="graphscope@alibaba-inc.com", - url="https://github.com/alibaba/GraphScope", - license="BSD", - platforms="Linux, Mac OS X, Windows", - keywords=["Jupyter", "Widgets", "IPython"], - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: BSD License", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.4", - "Programming Language :: Python :: 3.5", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Framework :: Jupyter", - ], - include_package_data=True, - install_requires=[ - "ipywidgets>=7.0.0", - "spectate>=0.4.1", - "traitlets", - "networkx", - ], - extras_require={ - "test": ["pytest>4.6", "pytest-cov", "nbval", "pandas"], - "examples": [ - "pandas" - # Any requirements for the examples to run - ], - "docs": [ - "sphinx", - "sphinx_rtd_theme", - "sphinx-autobuild>=2020.9.1", - "jupyter-sphinx>=0.3.1", - "sphinx-copybutton", - "nbsphinx", - "nbsphinx-link", - "networkx", - "pandas", - ], - }, - entry_points={}, -) - -if __name__ == "__main__": - setup(**setup_args) diff --git a/python/jupyter/graphscope/setupbase.py b/python/jupyter/graphscope/setupbase.py deleted file mode 100644 index ef68a4ca12a1..000000000000 --- a/python/jupyter/graphscope/setupbase.py +++ /dev/null @@ -1,707 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -""" -This file originates from the 'jupyter-packaging' package, and -contains a set of useful utilities for including npm packages -within a Python package. -""" -import functools -import io -import os -import pipes -import re -import shlex -import subprocess -import sys -from collections import defaultdict -from os.path import join as pjoin - -# BEFORE importing distutils, remove MANIFEST. distutils doesn't properly -# update it when the contents of directories change. -if os.path.exists("MANIFEST"): - os.remove("MANIFEST") - - -from distutils import log -from distutils.cmd import Command -from distutils.command.build_py import build_py -from distutils.command.sdist import sdist - -from setuptools.command.bdist_egg import bdist_egg -from setuptools.command.develop import develop - -try: - from wheel.bdist_wheel import bdist_wheel -except ImportError: - bdist_wheel = None - -if sys.platform == "win32": - from subprocess import list2cmdline -else: - - def list2cmdline(cmd_list): - return " ".join(map(pipes.quote, cmd_list)) - - -__version__ = "0.2.0" - -# --------------------------------------------------------------------------- -# Top Level Variables -# --------------------------------------------------------------------------- - -HERE = os.path.abspath(os.path.dirname(__file__)) -is_repo = os.path.exists(pjoin(HERE, ".git")) -node_modules = pjoin(HERE, "node_modules") - -SEPARATORS = os.sep if os.altsep is None else os.sep + os.altsep - -npm_path = ":".join( - [ - pjoin(HERE, "node_modules", ".bin"), - os.environ.get("PATH", os.defpath), - ] -) - -if "--skip-npm" in sys.argv: - print("Skipping npm install as requested.") - skip_npm = True - sys.argv.remove("--skip-npm") -else: - skip_npm = False - - -# --------------------------------------------------------------------------- -# Public Functions -# --------------------------------------------------------------------------- -def find_packages(top=HERE): - """ - Find all of the packages. - """ - packages = [] - for d, dirs, _ in os.walk(top, followlinks=True): - if os.path.exists(pjoin(d, "__init__.py")): - packages.append(os.path.relpath(d, top).replace(os.path.sep, ".")) - elif d != top: - # Don't look for packages in subfolders if current is not a package - dirs[:] = [] - return packages - - -def update_package_data(distribution): - """update build_py options to get package_data changes""" - build_py = distribution.get_command_obj("build_py") - build_py.finalize_options() - - -class bdist_egg_disabled(bdist_egg): - """Disabled version of bdist_egg - - Prevents setup.py install performing setuptools' default easy_install, - which it should never ever do. - """ - - def run(self): - sys.exit( - "Aborting implicit building of eggs. Use `pip install .` " - " to install from source." - ) - - -def create_cmdclass(prerelease_cmd=None, package_data_spec=None, data_files_spec=None): - """Create a command class with the given optional prerelease class. - - Parameters - ---------- - prerelease_cmd: (name, Command) tuple, optional - The command to run before releasing. - package_data_spec: dict, optional - A dictionary whose keys are the dotted package names and - whose values are a list of glob patterns. - data_files_spec: list, optional - A list of (path, dname, pattern) tuples where the path is the - `data_files` install path, dname is the source directory, and the - pattern is a glob pattern. - - Notes - ----- - We use specs so that we can find the files *after* the build - command has run. - - The package data glob patterns should be relative paths from the package - folder containing the __init__.py file, which is given as the package - name. - e.g. `dict(foo=['./bar/*', './baz/**'])` - - The data files directories should be absolute paths or relative paths - from the root directory of the repository. Data files are specified - differently from `package_data` because we need a separate path entry - for each nested folder in `data_files`, and this makes it easier to - parse. - e.g. `('share/foo/bar', 'pkgname/bizz, '*')` - """ - wrapped = [prerelease_cmd] if prerelease_cmd else [] - if package_data_spec or data_files_spec: - wrapped.append("handle_files") - wrapper = functools.partial(_wrap_command, wrapped) - handle_files = _get_file_handler(package_data_spec, data_files_spec) - - if "bdist_egg" in sys.argv: - egg = wrapper(bdist_egg, strict=True) - else: - egg = bdist_egg_disabled - - cmdclass = dict( - build_py=wrapper(build_py, strict=is_repo), - bdist_egg=egg, - sdist=wrapper(sdist, strict=True), - handle_files=handle_files, - ) - - if bdist_wheel: - cmdclass["bdist_wheel"] = wrapper(bdist_wheel, strict=True) - - cmdclass["develop"] = wrapper(develop, strict=True) - return cmdclass - - -def command_for_func(func): - """Create a command that calls the given function.""" - - class FuncCommand(BaseCommand): - def run(self): - func() - update_package_data(self.distribution) - - return FuncCommand - - -def run(cmd, **kwargs): - """Echo a command before running it. Defaults to repo as cwd""" - log.info("> %s", list2cmdline(cmd)) - kwargs.setdefault("cwd", HERE) - kwargs.setdefault("shell", os.name == "nt") - if not isinstance(cmd, (list, tuple)) and os.name != "nt": - cmd = shlex.split(cmd) - cmd_path = which(cmd[0]) - if not cmd_path: - sys.exit( - "Aborting. Could not find cmd (%s) in path. " - "If command is not expected to be in user's path, " - "use an absolute path." % cmd[0] - ) - cmd[0] = cmd_path - return subprocess.check_call(cmd, **kwargs) - - -def is_stale(target, source): - """Test whether the target file/directory is stale based on the source - file/directory. - """ - if not os.path.exists(target): - return True - target_mtime = recursive_mtime(target) or 0 - return compare_recursive_mtime(source, cutoff=target_mtime) - - -class BaseCommand(Command): - """Empty command because Command needs subclasses to override too much""" - - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def get_inputs(self): - return [] - - def get_outputs(self): - return [] - - -def combine_commands(*commands): - """Return a Command that combines several commands.""" - - class CombinedCommand(Command): - user_options = [] - - def initialize_options(self): - self.commands = [] - for C in commands: - self.commands.append(C(self.distribution)) - for c in self.commands: - c.initialize_options() - - def finalize_options(self): - for c in self.commands: - c.finalize_options() - - def run(self): - for c in self.commands: - c.run() - - return CombinedCommand - - -def compare_recursive_mtime(path, cutoff, newest=True): - """Compare the newest/oldest mtime for all files in a directory. - - Cutoff should be another mtime to be compared against. If an mtime that is - newer/older than the cutoff is found it will return True. - E.g. if newest=True, and a file in path is newer than the cutoff, it will - return True. - """ - if os.path.isfile(path): - mt = mtime(path) - if newest: - if mt > cutoff: - return True - elif mt < cutoff: - return True - for dirname, _, filenames in os.walk(path, topdown=False): - for filename in filenames: - mt = mtime(pjoin(dirname, filename)) - if newest: # Put outside of loop? - if mt > cutoff: - return True - elif mt < cutoff: - return True - return False - - -def recursive_mtime(path, newest=True): - """Gets the newest/oldest mtime for all files in a directory.""" - if os.path.isfile(path): - return mtime(path) - current_extreme = None - for dirname, dirnames, filenames in os.walk(path, topdown=False): - for filename in filenames: - mt = mtime(pjoin(dirname, filename)) - if newest: # Put outside of loop? - if mt >= (current_extreme or mt): - current_extreme = mt - elif mt <= (current_extreme or mt): - current_extreme = mt - return current_extreme - - -def mtime(path): - """shorthand for mtime""" - return os.stat(path).st_mtime - - -def install_npm( - path=None, - build_dir=None, - source_dir=None, - build_cmd="build", - force=False, - npm=None, -): - """Return a Command for managing an npm installation. - - Note: The command is skipped if the `--skip-npm` flag is used. - - Parameters - ---------- - path: str, optional - The base path of the node package. Defaults to the repo root. - build_dir: str, optional - The target build directory. If this and source_dir are given, - the JavaScript will only be build if necessary. - source_dir: str, optional - The source code directory. - build_cmd: str, optional - The npm command to build assets to the build_dir. - npm: str or list, optional. - The npm executable name, or a tuple of ['node', executable]. - """ - - class NPM(BaseCommand): - description = "install package.json dependencies using npm" - - def run(self): - if skip_npm: - log.info("Skipping npm-installation") - return - node_package = path or HERE - node_modules = pjoin(node_package, "node_modules") - is_yarn = os.path.exists(pjoin(node_package, "yarn.lock")) - - npm_cmd = npm - - if npm is None: - if is_yarn: - npm_cmd = ["yarn"] - else: - npm_cmd = ["npm"] - - if not which(npm_cmd[0]): - log.error( - "`%s` unavailable. If you're running this command " - "using sudo, make sure `%s` is available to sudo", - npm_cmd[0], - npm_cmd[0], - ) - return - - if force or is_stale(node_modules, pjoin(node_package, "package.json")): - log.info( - "Installing build dependencies with npm. This may " - "take a while..." - ) - run(npm_cmd + ["install"], cwd=node_package) - if build_dir and source_dir and not force: - should_build = is_stale(build_dir, source_dir) - else: - should_build = True - if should_build: - run(npm_cmd + ["run", build_cmd], cwd=node_package) - - return NPM - - -def ensure_targets(targets): - """Return a Command that checks that certain files exist. - - Raises a ValueError if any of the files are missing. - - Note: The check is skipped if the `--skip-npm` flag is used. - """ - - class TargetsCheck(BaseCommand): - def run(self): - if skip_npm: - log.info("Skipping target checks") - return - missing = [t for t in targets if not os.path.exists(t)] - if missing: - raise ValueError(("missing files: %s" % missing)) - - return TargetsCheck - - -# `shutils.which` function copied verbatim from the Python-3.3 source. -def which(cmd, mode=os.F_OK | os.X_OK, path=None): - """Given a command, mode, and a PATH string, return the path which - conforms to the given mode on the PATH, or None if there is no such - file. - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. - """ - - # Check that a given file can be accessed with the correct mode. - # Additionally check that `file` is not a directory, as on Windows - # directories pass the os.access check. - def _access_check(fn, mode): - return os.path.exists(fn) and os.access(fn, mode) and not os.path.isdir(fn) - - # Short circuit. If we're given a full path which matches the mode - # and it exists, we're done here. - if _access_check(cmd, mode): - return cmd - - path = (path or os.environ.get("PATH", os.defpath)).split(os.pathsep) - - if sys.platform == "win32": - # The current directory takes precedence on Windows. - if os.curdir not in path: - path.insert(0, os.curdir) - - # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - matches = [cmd for ext in pathext if cmd.lower().endswith(ext.lower())] - # If it does match, only test that one, otherwise we have to try - # others. - files = [cmd] if matches else [cmd + ext.lower() for ext in pathext] - else: - # On other platforms you don't have things like PATHEXT to tell you - # what file suffixes are executable, so just pass on cmd as-is. - files = [cmd] - - seen = set() - for d in path: - d = os.path.normcase(d) - if d not in seen: - seen.add(d) - for thefile in files: - name = os.path.join(d, thefile) - if _access_check(name, mode): - return name - return None - - -# --------------------------------------------------------------------------- -# Private Functions -# --------------------------------------------------------------------------- - - -def _wrap_command(cmds, cls, strict=True): - """Wrap a setup command - - Parameters - ---------- - cmds: list(str) - The names of the other commands to run prior to the command. - strict: boolean, optional - Wether to raise errors when a pre-command fails. - """ - - class WrappedCommand(cls): - def run(self): - if not getattr(self, "uninstall", None): - try: - [self.run_command(cmd) for cmd in cmds] - except Exception: - if strict: - raise - else: - pass - # update package data - update_package_data(self.distribution) - - result = cls.run(self) - return result - - return WrappedCommand - - -def _get_file_handler(package_data_spec, data_files_spec): - """Get a package_data and data_files handler command.""" - - class FileHandler(BaseCommand): - def run(self): - package_data = self.distribution.package_data - package_spec = package_data_spec or dict() - - for key, patterns in package_spec.items(): - package_data[key] = _get_package_data(key, patterns) - - self.distribution.data_files = _get_data_files( - data_files_spec, self.distribution.data_files - ) - - return FileHandler - - -def _glob_pjoin(*parts): - """Join paths for glob processing""" - if parts[0] in (".", ""): - parts = parts[1:] - return pjoin(*parts).replace(os.sep, "/") - - -def _get_data_files(data_specs, existing, top=HERE): - """Expand data file specs into valid data files metadata. - - Parameters - ---------- - data_specs: list of tuples - See [create_cmdclass] for description. - existing: list of tuples - The existing distrubution data_files metadata. - - Returns - ------- - A valid list of data_files items. - """ - # Extract the existing data files into a staging object. - file_data = defaultdict(list) - for path, files in existing or []: - file_data[path] = files - - # Extract the files and assign them to the proper data - # files path. - for path, dname, pattern in data_specs or []: - if os.path.isabs(dname): - dname = os.path.relpath(dname, top) - dname = dname.replace(os.sep, "/") - offset = 0 if dname in (".", "") else len(dname) + 1 - files = _get_files(_glob_pjoin(dname, pattern), top=top) - for fname in files: - # Normalize the path. - root = os.path.dirname(fname) - full_path = _glob_pjoin(path, root[offset:]) - print(dname, root, full_path, offset) - if full_path.endswith("/"): - full_path = full_path[:-1] - file_data[full_path].append(fname) - - # Construct the data files spec. - data_files = [] - for path, files in file_data.items(): - data_files.append((path, files)) - return data_files - - -def _get_files(file_patterns, top=HERE): - """Expand file patterns to a list of paths. - - Parameters - ----------- - file_patterns: list or str - A list of glob patterns for the data file locations. - The globs can be recursive if they include a `**`. - They should be relative paths from the top directory or - absolute paths. - top: str - the directory to consider for data files - - Note: - Files in `node_modules` are ignored. - """ - if not isinstance(file_patterns, (list, tuple)): - file_patterns = [file_patterns] - - for i, p in enumerate(file_patterns): - if os.path.isabs(p): - file_patterns[i] = os.path.relpath(p, top) - - matchers = [_compile_pattern(p) for p in file_patterns] - - files = set() - - for root, dirnames, filenames in os.walk(top): - # Don't recurse into node_modules - if "node_modules" in dirnames: - dirnames.remove("node_modules") - for m in matchers: - for filename in filenames: - fn = os.path.relpath(_glob_pjoin(root, filename), top) - fn = fn.replace(os.sep, "/") - if m(fn): - files.add(fn.replace(os.sep, "/")) - - return list(files) - - -def _get_package_data(root, file_patterns=None): - """Expand file patterns to a list of `package_data` paths. - - Parameters - ----------- - root: str - The relative path to the package root from `HERE`. - file_patterns: list or str, optional - A list of glob patterns for the data file locations. - The globs can be recursive if they include a `**`. - They should be relative paths from the root or - absolute paths. If not given, all files will be used. - - Note: - Files in `node_modules` are ignored. - """ - if file_patterns is None: - file_patterns = ["*"] - return _get_files(file_patterns, _glob_pjoin(HERE, root)) - - -def _compile_pattern(pat, ignore_case=True): - """Translate and compile a glob pattern to a regular expression matcher.""" - if isinstance(pat, bytes): - pat_str = pat.decode("ISO-8859-1") - res_str = _translate_glob(pat_str) - res = res_str.encode("ISO-8859-1") - else: - res = _translate_glob(pat) - flags = re.IGNORECASE if ignore_case else 0 - return re.compile(res, flags=flags).match - - -def _iexplode_path(path): - """Iterate over all the parts of a path. - - Splits path recursively with os.path.split(). - """ - (head, tail) = os.path.split(path) - if not head or (not tail and head == path): - if head: - yield head - if tail or not head: - yield tail - return - for p in _iexplode_path(head): - yield p - yield tail - - -def _translate_glob(pat): - """Translate a glob PATTERN to a regular expression.""" - translated_parts = [] - for part in _iexplode_path(pat): - translated_parts.append(_translate_glob_part(part)) - os_sep_class = "[%s]" % re.escape(SEPARATORS) - res = _join_translated(translated_parts, os_sep_class) - return "{res}\\Z(?ms)".format(res=res) - - -def _join_translated(translated_parts, os_sep_class): - """Join translated glob pattern parts. - - This is different from a simple join, as care need to be taken - to allow ** to match ZERO or more directories. - """ - res = "" - for part in translated_parts[:-1]: - if part == ".*": - # drop separator, since it is optional - # (** matches ZERO or more dirs) - res += part - else: - res += part + os_sep_class - - if translated_parts[-1] == ".*": - # Final part is ** - res += ".+" - # Follow stdlib/git convention of matching all sub files/directories: - res += "({os_sep_class}?.*)?".format(os_sep_class=os_sep_class) - else: - res += translated_parts[-1] - return res - - -def _translate_glob_part(pat): - """Translate a glob PATTERN PART to a regular expression.""" - # Code modified from Python 3 standard lib fnmatch: - if pat == "**": - return ".*" - i, n = 0, len(pat) - res = [] - while i < n: - c = pat[i] - i = i + 1 - if c == "*": - # Match anything but path separators: - res.append("[^%s]*" % SEPARATORS) - elif c == "?": - res.append("[^%s]?" % SEPARATORS) - elif c == "[": - j = i - if j < n and pat[j] == "!": - j = j + 1 - if j < n and pat[j] == "]": - j = j + 1 - while j < n and pat[j] != "]": - j = j + 1 - if j >= n: - res.append("\\[") - else: - stuff = pat[i:j].replace("\\", "\\\\") - i = j + 1 - if stuff[0] == "!": - stuff = "^" + stuff[1:] - elif stuff[0] == "^": - stuff = "\\" + stuff - res.append("[%s]" % stuff) - else: - res.append(re.escape(c)) - return "".join(res) diff --git a/python/jupyter/graphscope/src/extension.ts b/python/jupyter/graphscope/src/extension.ts deleted file mode 100644 index b3459b2a11f3..000000000000 --- a/python/jupyter/graphscope/src/extension.ts +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -(window as any).__webpack_public_path__ = - document.querySelector('body')!.getAttribute('data-base-url') + - 'nbextensions/graphscope-jupyter'; - -export * from './index'; diff --git a/python/jupyter/graphscope/src/graph.ts b/python/jupyter/graphscope/src/graph.ts deleted file mode 100644 index 2ac21e9d0246..000000000000 --- a/python/jupyter/graphscope/src/graph.ts +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import '../css/index.css'; -import { - ISerializers, - WidgetModel, - DOMWidgetView -} from '@jupyter-widgets/base'; -import React from 'react'; -import ReactDOM from 'react-dom'; -import { GraphScopeComponent } from '@antv/graphin-graphscope'; - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const widgets = require('@jupyter-widgets/base'); - -import { MODULE_NAME, MODULE_VERSION } from './version'; - -export class GraphModel extends WidgetModel { - defaults() { - return { - ...super.defaults(), - _model_name: 'GraphModel', - _model_module: GraphModel.model_module, - _model_module_version: GraphModel.model_module_version, - nodes: [], - edges: [], - value: '', - zoom: false - }; - } - - static serializers: ISerializers = { - nodes: { deserialize: widgets.unpack_models }, - edges: { deserialize: widgets.unpack_models }, - ...WidgetModel.serializers - }; - - static model_module = MODULE_NAME; - static model_module_version = MODULE_VERSION; -} - -export class GraphView extends DOMWidgetView { - private dom: HTMLDivElement; - private graphData: any; - - constructor(params: any) { - super({ - model: params.model, - options: params.options - }); - - this.el.addEventListener('contextmenu', this.onClick.bind(this)); - } - - onClick(evt: any) { - evt.stopPropagation(); - evt.preventDefault(); - this.send({ - type: 'contextmenu', - params: evt - }); - } - - valueChanged() { - const value = this.model.get('value'); - - console.log('change value', value); - - if (value) { - const currentData = JSON.parse(value); - console.log('json parse value', currentData); - this.graphData = currentData; - this.renderGraph(currentData); - } - } - - render() { - //Python attributes that must be sync. with frontend - this.model.on('change:value', this.valueChanged, this); - - if (this.dom) { - ReactDOM.unmountComponentAtNode(this.dom); - } else { - this.valueChanged(); - } - } - - queryNeighbors(nodeId: string, degree: number) { - const model = this.graphData.nodes.filter( - (node: any) => node.id === nodeId - ); - - console.log('queryNeighbors', nodeId, degree, model); - this.send({ - nodeId, - degree - }); - // setData({ - // nodes: [...this.graphData.nodes, ...newData.nodes], - // edges: [...this.graphData.edges, ...newData.edges] - // } as any); - } - - handleNodeClick(model: any, type: string) { - console.log('click node', model); - } - - renderGraph(data: any) { - // dom δΈε­˜εœ¨ζ—ΆοΌŒεˆ›ε»Ί dom ε…ƒη΄ εΉΆζ·»εŠ εˆ° this.el δΈ­ - if (!this.dom) { - this.el.classList.add('custom-widget'); - this.dom = document.createElement('div'); - this.dom.style.width = '100%'; - this.dom.style.height = '400px'; - this.dom.style.position = 'relative'; - - this.el.append(this.dom); - } - - console.log('zoomcanvas', this.model.get('zoom')); - - setTimeout(() => { - ReactDOM.render( - React.createElement( - GraphScopeComponent, - { - graphDOM: this.dom, - // width: 1000, - height: 400, - neighbors: this.queryNeighbors.bind(this), - data: data, - hasMinimap: true, - hasContextMenu: true, - zoomCanvas: this.model.get('zoom') - }, - null - ), - this.el - ); - }, 16); - } -} diff --git a/python/jupyter/graphscope/src/index.ts b/python/jupyter/graphscope/src/index.ts deleted file mode 100644 index 6025784a2c3b..000000000000 --- a/python/jupyter/graphscope/src/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -export * from './version'; -export * from './graph'; diff --git a/python/jupyter/graphscope/src/plugin.ts b/python/jupyter/graphscope/src/plugin.ts deleted file mode 100644 index c156a3ce615a..000000000000 --- a/python/jupyter/graphscope/src/plugin.ts +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -import { Application, IPlugin } from '@phosphor/application'; - -import { Widget } from '@phosphor/widgets'; - -import { IJupyterWidgetRegistry } from '@jupyter-widgets/base'; - -import { MODULE_NAME, MODULE_VERSION } from './version'; - -const EXTENSION_ID = 'graphscope-jupyter:plugin'; - -/** - * The example plugin. - */ -const examplePlugin: IPlugin, void> = { - id: EXTENSION_ID, - requires: [IJupyterWidgetRegistry as any], - activate: activateWidgetExtension, - autoStart: true, -}; - -export default examplePlugin; - -/** - * Activate the widget extension. - */ -function activateWidgetExtension( - app: Application, - registry: IJupyterWidgetRegistry -): void { - registry.registerWidget({ - name: MODULE_NAME, - version: MODULE_VERSION, - - exports: async () => - await import(/* webpackChunkName: "graphscope-jupyter" */ './index'), - }); -} diff --git a/python/jupyter/graphscope/src/version.ts b/python/jupyter/graphscope/src/version.ts deleted file mode 100644 index 7fed0b04b016..000000000000 --- a/python/jupyter/graphscope/src/version.ts +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -// eslint-disable-next-line @typescript-eslint/no-var-requires -const data = require('../package.json'); - -export const MODULE_VERSION = data.version; - -/* - * The current package name. - */ -export const MODULE_NAME = data.name; diff --git a/python/jupyter/graphscope/tsconfig.json b/python/jupyter/graphscope/tsconfig.json deleted file mode 100644 index 415c82431a32..000000000000 --- a/python/jupyter/graphscope/tsconfig.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "compilerOptions": { - "declaration": true, - "esModuleInterop": true, - "lib": ["es2015", "dom"], - "jsx": "react", - "module": "esnext", - "moduleResolution": "node", - "noEmitOnError": true, - "noUnusedLocals": true, - "outDir": "lib", - "resolveJsonModule": true, - "rootDir": "src", - "skipLibCheck": true, - "sourceMap": true, - "strict": true, - "target": "es2015", - "strictPropertyInitialization": false - }, - "include": ["src/*"] -} diff --git a/python/jupyter/graphscope/webpack.config.js b/python/jupyter/graphscope/webpack.config.js deleted file mode 100644 index e73258f90a7e..000000000000 --- a/python/jupyter/graphscope/webpack.config.js +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -const path = require('path'); -const version = require('./package.json').version; - -// Custom webpack rules -const rules = [ - { test: /\.ts$/, loader: 'ts-loader' }, - { test: /\.js$/, loader: 'source-map-loader' }, - { test: /\.css$/, use: ['style-loader', 'css-loader'] } -]; - -// Packages that shouldn't be bundled but loaded at runtime -const externals = ['@jupyter-widgets/base']; - -const resolve = { - // Add '.ts' and '.tsx' as resolvable extensions. - extensions: ['.webpack.js', '.web.js', '.ts', '.js', '.tsx'] -}; - -module.exports = [ - /** - * Notebook extension - * - * This bundle only contains the part of the JavaScript that is run on load of - * the notebook. - */ - { - entry: './src/extension.ts', - output: { - filename: 'index.js', - path: path.resolve(__dirname, 'graphscope_jupyter', 'nbextension', 'static'), - libraryTarget: 'umd' - }, - module: { - rules: rules - }, - devtool: 'source-map', - externals, - resolve - }, - - /** - * Embeddable graphscope-jupyter bundle - * - * This bundle is almost identical to the notebook extension bundle. The only - * difference is in the configuration of the webpack public path for the - * static assets. - * - * The target bundle is always `dist/index.js`, which is the path required by - * the custom widget embedder. - */ - { - entry: './src/index.ts', - output: { - filename: 'index.js', - path: path.resolve(__dirname, 'dist'), - libraryTarget: 'umd', - library: 'graphscope-jupyter', - publicPath: 'https://unpkg.com/graphscope-jupyter@' + version + '/dist/' - }, - devtool: 'source-map', - module: { - rules: rules - }, - externals, - resolve - }, - - /** - * Documentation widget bundle - * - * This bundle is used to embed widgets in the package documentation. - */ - { - entry: './src/index.ts', - output: { - filename: 'embed-bundle.js', - path: path.resolve(__dirname, 'docs', 'source', '_static'), - library: 'graphscope-jupyter', - libraryTarget: 'umd' - }, - module: { - rules: rules - }, - devtool: 'source-map', - externals, - resolve - } -]; diff --git a/python/setup.py b/python/setup.py index 6910fbf4d35e..25824fa68319 100644 --- a/python/setup.py +++ b/python/setup.py @@ -251,19 +251,20 @@ def run(self): graphlearn_shared_lib = "libgraphlearn_shared.dylib" else: graphlearn_shared_lib = "libgraphlearn_shared.so" - if not os.path.isfile( - os.path.join( - pkg_root, - "..", - "learning_engine", - "graph-learn", - "graphlearn", - "built", - "lib", - graphlearn_shared_lib, - ) - ): - raise ValueError("You must build the graphlearn library at first") + if os.environ.get("WITHOUT_LEARNING_ENGINE", None) is None: + if not os.path.isfile( + os.path.join( + pkg_root, + "..", + "learning_engine", + "graph-learn", + "graphlearn", + "built", + "lib", + graphlearn_shared_lib, + ) + ): + raise ValueError("You must build the graphlearn library at first") self.run_command("build_proto") bdist_wheel.run(self) @@ -313,6 +314,9 @@ def parsed_package_data(): def build_learning_engine(): + if os.environ.get("WITHOUT_LEARNING_ENGINE", None) is not None: + return [] + ext_modules = [graphlearn_ext()] if torch and os.path.exists(os.path.join(glt_root_path, "graphlearn_torch")): sys.path.insert(