From 1130f91167acec63ce83d8be4f89f87fb610b7eb Mon Sep 17 00:00:00 2001 From: Dongze Li Date: Wed, 18 Sep 2024 11:16:06 +0800 Subject: [PATCH] ci: integrate install dependencies script of Interactive into gsctl (#4234) ```bash # pip3 install gsctl gsctl install-deps dev-interactive # or pip3 install click cd GraphScope python3 gsctl.py install-deps dev-interactive ``` --- .github/workflows/interactive.yml | 10 +- flex/scripts/install_dependencies.sh | 74 ------------ .../graphscope/gsctl/scripts/install_deps.sh | 108 +++++++++++++----- 3 files changed, 86 insertions(+), 106 deletions(-) delete mode 100755 flex/scripts/install_dependencies.sh diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 640ccd03d8f1..8610723964db 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -10,14 +10,14 @@ on: paths: - 'flex/**' - 'interactive_engine/**' - - '.github/workflows/hqps-db-ci.yml' + - '.github/workflows/interactive.yml' pull_request: branches: - main paths: - 'flex/**' - 'interactive_engine/**' - - '.github/workflows/hqps-db-ci.yml' + - '.github/workflows/interactive.yml' concurrency: group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }} @@ -328,8 +328,9 @@ jobs: - name: Install dependencies run: | - cd ${GITHUB_WORKSPACE}/flex - bash ./scripts/install_dependencies.sh 4 + cd ${GITHUB_WORKSPACE} + python3 -m pip install --upgrade pip && python3 -m pip install click + python3 ./gsctl.py install-deps dev-interactive - name: Setup tmate session if: false @@ -337,6 +338,7 @@ jobs: - name: Build run: | + source ${HOME}/.graphscope_env cd ${GITHUB_WORKSPACE}/flex git submodule update --init mkdir build && cd build # only test default build diff --git a/flex/scripts/install_dependencies.sh b/flex/scripts/install_dependencies.sh deleted file mode 100755 index 22a299c7907f..000000000000 --- a/flex/scripts/install_dependencies.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -set -e -# accept at most one argument, parallelism -if [ $# -gt 1 ]; then - echo "Usage: $0 [parallelism]" - exit 1 -fi -parallelism=$(nproc) -if [ $# -eq 1 ]; then - parallelism=$1 -fi -echo "parallelism: $parallelism" - -sudo apt-get update && sudo apt install -y \ - ninja-build ragel libhwloc-dev libnuma-dev libpciaccess-dev vim wget curl \ - git g++ libunwind-dev libgoogle-glog-dev cmake libopenmpi-dev default-jdk libcrypto++-dev \ - libxml2-dev protobuf-compiler libprotobuf-dev libncurses5-dev libcurl4-openssl-dev -sudo apt install -y xfslibs-dev libgnutls28-dev liblz4-dev maven openssl pkg-config rapidjson-dev \ - libsctp-dev gcc make python3 systemtap-sdt-dev libtool libyaml-cpp-dev \ - libc-ares-dev stow libfmt-dev diffutils valgrind doxygen python3-pip net-tools graphviz - -# install boost -pushd /tmp/ -curl -L -o boost_1_75_0.tar.gz "https://graphscope.oss-cn-beijing.aliyuncs.com/dependencies/boost_1_75_0.tar.gz" -tar -xzf boost_1_75_0.tar.gz -pushd boost_1_75_0 && ./bootstrap.sh --with-libraries=system,filesystem,context,atomic,program_options,regex,thread,chrono,date_time,test # unit_test_framework used by seastar -sudo ./b2 install link=shared runtime-link=shared variant=release threading=multi -popd && sudo rm -rf boost_1_75_0 -rm boost_1_75_0.tar.gz - -pushd /tmp -git clone https://github.com/alibaba/libgrape-lite.git -cd libgrape-lite -git checkout v0.3.2 -mkdir build && cd build && cmake .. -DBUILD_LIBGRAPELITE_TESTS=OFF -make -j ${parallelism} && sudo make install -sudo cp /usr/local/lib/libgrape-lite.so /usr/lib/libgrape-lite.so -popd && rm -rf /tmp/libgrape-lite - -pushd /tmp && sudo apt-get install -y -V ca-certificates lsb-release wget -curl -o apache-arrow-apt-source-latest.deb https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -if ! sudo apt-get install -y ./apache-arrow-apt-source-latest.deb; then - pushd /tmp - git clone https://github.com/apache/arrow.git --single-branch --branch apache-arrow-15.0.2-1 - cd arrow/cpp && mkdir build && cd build && cmake .. -DARROW_CSV=ON && make -j ${parallelism} && sudo make install - popd && rm -r /tmp/arrow/ -else - sudo apt-get update && sudo apt-get install -y libarrow-dev=15.0.2-1 -fi -popd && rm -rf /tmp/apache-arrow-apt-source-latest.deb - -# install opentelemetry -cd /tmp -git clone https://github.com/open-telemetry/opentelemetry-cpp -b v1.15.0 -cd opentelemetry-cpp -cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=17 \ --DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \ --DWITH_OTLP_HTTP=ON -DWITH_OTLP_GRPC=OFF \ --DWITH_ABSEIL=OFF -DWITH_PROMETHEUS=OFF \ --DBUILD_TESTING=OFF -DWITH_EXAMPLES=OFF -make -j ${parallelism} && sudo make install -cd /tmp && rm -rf /tmp/opentelemetry-cpp - -pushd /tmp -git clone https://github.com/alibaba/hiactor.git -b v0.1.1 --single-branch -cd hiactor && git submodule update --init --recursive -sudo bash ./seastar/seastar/install-dependencies.sh -mkdir build && cd build -cmake -DHiactor_DEMOS=OFF -DHiactor_TESTING=OFF -DHiactor_DPDK=OFF -DHiactor_CXX_DIALECT=gnu++17 -DSeastar_CXX_FLAGS="-DSEASTAR_DEFAULT_ALLOCATOR -mno-avx512" .. -make -j ${parallelism} && sudo make install -popd && rm -rf /tmp/hiactor - -sudo sh -c 'echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf' -sudo sysctl -p /etc/sysctl.conf diff --git a/python/graphscope/gsctl/scripts/install_deps.sh b/python/graphscope/gsctl/scripts/install_deps.sh index a2bdd265a918..ed737c0f2dd8 100755 --- a/python/graphscope/gsctl/scripts/install_deps.sh +++ b/python/graphscope/gsctl/scripts/install_deps.sh @@ -347,8 +347,25 @@ install_openssl_static() { rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" } +# arrow for ubuntu and centos +install_arrow() { + if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then + if ! dpkg -s libarrow-dev &>/dev/null; then + ${SUDO} apt-get install -y lsb-release + # shellcheck disable=SC2046,SC2019,SC2018 + wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/ + ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb + ${SUDO} apt-get update -y + ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1 + rm /tmp/apache-arrow-apt-source-latest-*.deb + fi + elif [[ "${OS_PLATFORM}" == *"CentOS"* || "${OS_PLATFORM}" == *"Aliyun"* ]]; then + install_arrow_centos + fi +} + # arrow for centos -install_apache_arrow() { +install_arrow_centos() { if [[ -f "${install_prefix}/include/arrow/api.h" ]]; then return 0 fi @@ -470,6 +487,28 @@ install_zlib() { rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" } +# opentelemetry +install_opentelemetry() { + pushd "${tempdir}" || exit + git clone https://github.com/open-telemetry/opentelemetry-cpp -b v1.15.0 + cd opentelemetry-cpp + cmake . -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DWITH_OTLP_HTTP=ON \ + -DWITH_OTLP_GRPC=OFF \ + -DWITH_ABSEIL=OFF \ + -DWITH_PROMETHEUS=OFF \ + -DBUILD_TESTING=OFF \ + -DWITH_EXAMPLES=OFF + make -j -j$(nproc) + make install + popd || exit + rm -rf "${tempdir:?}/opentelemetry-cpp" +} + # grpc for centos install_grpc() { if [[ -f "${install_prefix}/include/grpcpp/grpcpp.h" ]]; then @@ -533,6 +572,24 @@ install_patchelf() { rm -rf "${tempdir:?}/${directory:?}" "${tempdir:?}/${file:?}" } +# libgrape-lite +install_libgrape_lite() { + if [[ -f "${install_prefix}/include/grape/grape.h" ]]; then + return 0 + fi + local branch=$1 + pushd "${tempdir}" || exit + git clone -b ${branch} https://github.com/alibaba/libgrape-lite.git + cd libgrape-lite + cmake . -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${install_prefix}" \ + -DBUILD_LIBGRAPELITE_TESTS=OFF + make -j$(nproc) + make install + popd || exit + rm -rf "${tempdir:?}/libgrape-lite" +} + # vineyard install_vineyard() { if [[ -f "${V6D_PREFIX}/include/vineyard/client/client.h" ]]; then @@ -705,7 +762,7 @@ ANALYTICAL_CENTOS_8=("${ANALYTICAL_CENTOS_7[@]}" "boost-devel" "gflags-devel" "g install_analytical_centos_common_dependencies() { install_patchelf - install_apache_arrow + install_arrow_centos install_openmpi install_protobuf install_zlib @@ -723,15 +780,7 @@ install_analytical_dependencies() { # patchelf install_patchelf # arrow - if ! dpkg -s libarrow-dev &>/dev/null; then - ${SUDO} apt-get install -y lsb-release - # shellcheck disable=SC2046,SC2019,SC2018 - wget -c https://apache.jfrog.io/artifactory/arrow/"$(lsb_release --id --short | tr 'A-Z' 'a-z')"/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb -P /tmp/ - ${SUDO} apt-get install -y -V /tmp/apache-arrow-apt-source-latest-"$(lsb_release --codename --short)".deb - ${SUDO} apt-get update -y - ${SUDO} apt-get install -y libarrow-dev=${ARROW_VERSION}-1 libarrow-dataset-dev=${ARROW_VERSION}-1 libarrow-acero-dev=${ARROW_VERSION}-1 libparquet-dev=${ARROW_VERSION}-1 - rm /tmp/apache-arrow-apt-source-latest-*.deb - fi + install_arrow # install boost >= 1.75 for leaf install_boost else @@ -782,20 +831,29 @@ install_analytical_java_dependencies() { fi } -FLEX_INTERACTIVE_UBUNTU=("rapidjson-dev") -FLEX_INTERACTIVE_CENTOS=("rapidjson-devel") +INTERACTIVE_MACOS=("apache-arrow" "rapidjson" "boost" "glog" "gflags") +INTERACTIVE_UBUNTU=("rapidjson-dev" "libgoogle-glog-dev" "libgflags-dev") +INTERACTIVE_CENTOS=("rapidjson-devel") -install_packages_for_interactive() { +install_interactive_dependencies() { + # dependencies package if [[ "${OS_PLATFORM}" == *"Darwin"* ]]; then - brew install ${FLEX_INTERACTIVE_MACOS[*]} + brew install ${INTERACTIVE_MACOS[*]} elif [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${FLEX_INTERACTIVE_UBUNTU[*]} + DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC ${SUDO} apt-get install -y ${INTERACTIVE_UBUNTU[*]} + install_arrow + install_boost + # hiactor is only supported on ubuntu + install_hiactor + ${SUDO} sh -c 'echo "fs.aio-max-nr = 1048576" >> /etc/sysctl.conf' + ${SUDO} sysctl -p /etc/sysctl.conf else - ${SUDO} yum install -y ${FLEX_INTERACTIVE_CENTOS[*]} + ${SUDO} yum install -y ${INTERACTIVE_CENTOS[*]} + install_arrow + install_boost fi -} - -install_interactive_dependencies() { + # libgrape-lite + install_libgrape_lite "v0.3.2" # java install_java_and_maven # rust @@ -806,14 +864,8 @@ install_interactive_dependencies() { rustup default 1.71.0 rustc --version fi - # hiactor - if [[ "${OS_PLATFORM}" == *"Ubuntu"* ]]; then - install_hiactor - else - warning "Skip installing dependencies for flex interactive on ${OS_PLATFORM}." - fi - # install rapidjson - install_packages_for_interactive + # opentelemetry + install_opentelemetry } install_learning_dependencies() {