fix: install-deps fails on macOS(intel) platform (#4266)

alibaba · Sep 29, 2024 · 47dbed3 · 47dbed3
1 parent a8d0975
commit 47dbed3
Show file tree

Hide file tree

Showing 3 changed files with 69 additions and 116 deletions.
diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml
@@ -19,87 +19,41 @@ concurrency:
 jobs:
   build-wheels:
     if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
-    runs-on: macos-12
+    runs-on: macos-13
     strategy:
       matrix:
-        python-version: ['3.11']
+        python-version: ['3.9']
 
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
-    - name: Install Dependencies
-      run: |
-        brew update || true
-        brew install bash coreutils
-
-        alias bash=$(brew --prefix)/bin/bash
-
-        export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
-
-        sudo mkdir /opt/graphscope
-        sudo chown -R $(id -u):$(id -g) /opt/graphscope
-
-        # Not sure when the first installation would fail, saying that the lock of python3.10 is taken
-        # However the second trial would success.
-        python3 -m pip install click
-        python3 gsctl.py install-deps dev || true
-        python3 gsctl.py install-deps dev
-        echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
-
     - name: Install JDK 11
       uses: actions/setup-java@v4
       with:
         distribution: 'zulu' # See 'Supported distributions' for available options
         java-version: '11'
 
-    - name: Reinstall glog
+    - name: Install Dependencies
       run: |
-        # Essential cause glog installed by brew is broken and produce corrupted messages
-        brew uninstall --ignore-dependencies glog
-        . ~/.graphscope_env
-        git clone https://github.com/google/glog.git -b v0.6.0
-        cd glog
-
-        cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF -DWITH_GTEST=OFF -DBUILD_TESTING=OFF
-        make -j
-        sudo make install
-
-        # make delocate-wheel happy
-        sudo mkdir -p /usr/local/opt/glog/lib
-        sudo ln -s /usr/local/lib/libglog*.dylib /usr/local/opt/glog/lib/
+        python3 -c 'import sys; print(sys.version_info[:])'
+        export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
 
-    - name: Install latest libgrape-lite
-      if: false
-      run: |
-        . ~/.graphscope_env
-        git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
-        cd /tmp/libgrape-lite
-        cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
-        make -j4
-        sudo make install
+        brew update || true
+        brew install bash coreutils
+        alias bash=$(brew --prefix)/bin/bash
 
-    - name: Install latest vineyard
-      if: false
-      run: |
-        source ~/.graphscope_env
-        python3 -m pip install libclang
-        git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
-        cd /tmp/v6d
-        git submodule update --init
-        cmake .  -DCMAKE_INSTALL_PREFIX=/usr/local \
-                 -DCMAKE_PREFIX_PATH=/usr/local \
-                 -DUSE_EXTERNAL_ETCD_LIBS=OFF \
-                 -DBUILD_SHARED_LIBS=ON \
-                 -DBUILD_VINEYARD_TESTS=OFF
-        make -j4
-        sudo make install
+        python3 -m pip install click
+        # install the specific vineyard version:
+        # python3 gsctl.py install-deps dev --v6d-version <branch or tag>
+        python3 gsctl.py install-deps dev
+        echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
 
     - name: Build Server Wheel
       run: |
@@ -115,8 +69,7 @@ jobs:
         # e.g. 0.15.0 -> 0.15.0a20220808
         time=$(date "+%Y%m%d")
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
-        if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]];
-        then
+        if [[ "${{ GITHUB.REF }}" == "refs/heads/main" ]]; then
           echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
         fi
 
@@ -143,58 +96,33 @@ jobs:
 
   build-client-wheels:
     if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch')
-    runs-on: macos-12
+    runs-on: macos-13
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install Dependencies
       run: |
+        python3 -c 'import sys; print(sys.version_info[:])'
+        export PATH=$HOME/.local/bin:$HOME/Library/Python/${{ matrix.python-version }}/bin:$PATH
+
         brew update || true
         brew install bash coreutils
         alias bash=$(brew --prefix)/bin/bash
-        sudo mkdir /opt/graphscope
-        sudo chown -R $(id -u):$(id -g) /opt/graphscope
 
         python3 -m pip install click
-        python3 gsctl.py install-deps dev || true
         python3 gsctl.py install-deps dev
-
-    - name: Install latest libgrape-lite
-      if: false
-      run: |
-        . ~/.graphscope_env
-        git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
-        cd /tmp/libgrape-lite
-        cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
-        make -j4
-        sudo make install
-
-    - name: Install latest vineyard
-      if: false
-      run: |
-        . ~/.graphscope_env
-        python3 -m pip install libclang
-        git clone --single-branch -b v0.23.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
-        cd /tmp/v6d
-        git submodule update --init
-        cmake .  -DCMAKE_INSTALL_PREFIX=/usr/local \
-                 -DCMAKE_PREFIX_PATH=/usr/local \
-                 -DUSE_EXTERNAL_ETCD_LIBS=OFF \
-                 -DBUILD_SHARED_LIBS=ON \
-                 -DBUILD_VINEYARD_TESTS=OFF
-        make -j4
-        sudo make install
+        echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
 
     - name: Build Client Wheels
       run: |
@@ -234,7 +162,7 @@ jobs:
     needs: [build-wheels, build-client-wheels]
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/download-artifact@v4
@@ -252,7 +180,7 @@ jobs:
         tar -zxf ./wheel-macos-${{ github.sha }}-${{ matrix.python-version }}/client.tar.gz
         mv ${GITHUB_WORKSPACE}/artifacts/python/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
 
-        if [ "$PYTHON" == "3.8" ]; then
+        if [ "$PYTHON" == "3.9" ]; then
           tar -zxf ./wheel-macos-${{ github.sha }}/graphscope.tar.gz
           mv ${GITHUB_WORKSPACE}/artifacts/coordinator/dist/wheelhouse/*.whl ${GITHUB_WORKSPACE}/upload_pypi/
         fi
@@ -277,12 +205,12 @@ jobs:
 
   python-test:
     if: ${{ (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'workflow_dispatch') }}
-    runs-on: macos-12
+    runs-on: macos-13
     needs: [build-wheels, build-client-wheels]
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11']
 
     steps:
     - uses: actions/download-artifact@v4

diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml
@@ -94,6 +94,7 @@ jobs:
     - name: Groot with GOpt Integration Test
       run: |
         . ${HOME}/.graphscope_env
+        rm -rf /tmp/gstest || true
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git /tmp/gstest
         mv /tmp/gstest/flex/ldbc-sf01-long-date /tmp/gstest/ldbc
         cp -r flex/interactive/examples/movies /tmp/gstest/

diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh
@@ -574,8 +574,14 @@ install_patchelf() {
 
 # libgrape-lite
 install_libgrape_lite() {
-  if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then
-    return 0
+  if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
+    if [[ -f "${HOMEBREW_PREFIX}/include/grape/grape.h" ]]; then
+      return 0
+    fi
+  else
+    if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then
+      return 0
+    fi
   fi
   local branch=$1
   pushd "${tempdir}" || exit
@@ -599,7 +605,13 @@ install_vineyard() {
   python3 -m pip --no-cache-dir install pip -U --user
   python3 -m pip --no-cache-dir install libclang wheel auditwheel --user
   auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')")
-  sed -i 's/p.error/logger.warning/g' ${auditwheel_path}
+  if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
+    BUILD_VINEYARD_GRAPH_WITH_GAR="OFF"
+    sed -i '' 's/p.error/logger.warning/g' ${auditwheel_path}
+  else
+    BUILD_VINEYARD_GRAPH_WITH_GAR="ON"
+    sed -i 's/p.error/logger.warning/g' ${auditwheel_path}
+  fi
   if [[ "${v6d_version}" != "v"* ]]; then
     directory="v6d"
     file="${directory}-${v6d_version}.tar.gz"
@@ -619,10 +631,14 @@ install_vineyard() {
         -DBUILD_VINEYARD_TESTS=OFF \
         -DBUILD_SHARED_LIBS=ON \
         -DBUILD_VINEYARD_PYTHON_BINDINGS=ON  \
-        -DBUILD_VINEYARD_GRAPH_WITH_GAR=ON
+        -DBUILD_VINEYARD_GRAPH_WITH_GAR=${BUILD_VINEYARD_GRAPH_WITH_GAR}
   make -j$(nproc)
   make install
-  strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard*
+  if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
+    strip "${V6D_PREFIX}"/bin/vineyard*
+  else
+    strip "${V6D_PREFIX}"/bin/vineyard* "${V6D_PREFIX}"/lib/libvineyard*
+  fi
   pip3 install --no-cache -i https://pypi.org/simple -U "vineyard" "vineyard-io"
   cp -rs "${V6D_PREFIX}"/* "${install_prefix}"/
   set +e
@@ -807,24 +823,30 @@ install_analytical_dependencies() {
   install_java_and_maven
   # install vineyard
   if [[ "${no_v6d}" != true ]]; then
-    if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
-      brew install vineyard
-    else
-      install_vineyard
-    fi
+    install_vineyard
+    # if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
+    #   brew install vineyard
+    # else
+    #   install_vineyard
+    # fi
   fi
 }
 
 install_analytical_java_dependencies() {
   # llvm
   if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
     brew install llvm || true # prevent the `brew link` failure
+    export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang
+    export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++
+    export CPPFLAGS="${CPPFLAGS} -I${HOMEBREW_PREFIX}/opt/llvm/include"
+    export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}
   elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then
     ${SUDO} apt-get install -y llvm-11-dev lld-11 clang-11
   else
     if [[ "${OS_VERSION}" -eq "7" ]]; then
       ${SUDO} yum install -y llvm-toolset-7.0-clang-devel
       source /opt/rh/llvm-toolset-7.0/enable
+      export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/
     else
       ${SUDO} yum install -y llvm-devel clang-devel lld
     fi
@@ -865,7 +887,9 @@ install_interactive_dependencies() {
     rustc --version
   fi
   # opentelemetry
-  install_opentelemetry
+  if [[ "${OS_PLATFORM}" != *"Darwin"* ]]; then
+    install_opentelemetry
+  fi
 }
 
 install_learning_dependencies() {
@@ -884,9 +908,9 @@ write_env_config() {
   } >> "${OUTPUT_ENV_FILE}"
   {
     if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
-      echo "export OPENSSL_ROOT_DIR=${homebrew_prefix}/opt/openssl"
-      echo "export OPENSSL_LIBRARIES=${homebrew_prefix}/opt/openssl/lib"
-      echo "export OPENSSL_SSL_LIBRARY=${homebrew_prefix}/opt/openssl/lib/libssl.dylib"
+      echo "export OPENSSL_ROOT_DIR=${HOMEBREW_PREFIX}/opt/openssl"
+      echo "export OPENSSL_LIBRARIES=${HOMEBREW_PREFIX}/opt/openssl/lib"
+      echo "export OPENSSL_SSL_LIBRARY=${HOMEBREW_PREFIX}/opt/openssl/lib/libssl.dylib"
     elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then
       if [[ "${OS_VERSION}" -eq "7" ]]; then
         echo "source /opt/rh/devtoolset-8/enable"
@@ -916,11 +940,11 @@ write_env_config() {
   {
     if [[ "${for_analytical_java}" == true ]]; then
       if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then
-        echo "export CC=${homebrew_prefix}/opt/llvm/bin/clang"
-        echo "export CXX=${homebrew_prefix}/opt/llvm/bin/clang++"
+        echo "export CC=${HOMEBREW_PREFIX}/opt/llvm/bin/clang"
+        echo "export CXX=${HOMEBREW_PREFIX}/opt/llvm/bin/clang++"
         echo "export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=${CC}"
-        echo "export LDFLAGS=\"-L${homebrew_prefix}/opt/llvm/lib\""
-        echo "export CPPFLAGS=\"-I${homebrew_prefix}/opt/llvm/include\""
+        echo "export LDFLAGS=\"-L${HOMEBREW_PREFIX}/opt/llvm/lib\""
+        echo "export CPPFLAGS=\"-I${HOMEBREW_PREFIX}/opt/llvm/include\""
       elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then
         echo "source /opt/rh/llvm-toolset-7.0/enable || true"
         echo "export LIBCLANG_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64/"