From 2f45939fee92a7181ace38def13b9c3721aae90f Mon Sep 17 00:00:00 2001 From: "joey.ljy" Date: Sun, 28 Jul 2024 09:18:17 +0800 Subject: [PATCH] Adapt setup scripts for the changes in upstream velox --- ep/build-velox/src/get_velox.sh | 26 +++++------ ep/build-velox/src/setup-centos7.sh | 18 +++----- ep/build-velox/src/setup-centos8.sh | 68 +++++++++-------------------- 3 files changed, 37 insertions(+), 75 deletions(-) diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 841787d64c23..2094affb5602 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -70,25 +70,22 @@ function process_setup_ubuntu { ensure_pattern_matched '\${SUDO} apt install -y libunwind-dev' scripts/setup-ubuntu.sh sed -i 's/${SUDO} apt install -y libunwind-dev//' scripts/setup-ubuntu.sh ensure_pattern_matched 'ccache' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh - sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh + sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh ensure_pattern_matched 'libgmock-dev' scripts/setup-ubuntu.sh sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake - ensure_pattern_matched 'github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive' scripts/setup-ubuntu.sh - sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive/wget_and_untar https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz boost \&\& cd boost/g' scripts/setup-ubuntu.sh ensure_pattern_matched 'function install_folly' scripts/setup-ubuntu.sh - sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh + sed -i '/^function install_folly.*/i function install_protobuf {\n wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf\n (\n cd protobuf\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n )\n}\n' scripts/setup-ubuntu.sh ensure_pattern_matched ' run_and_time install_folly' scripts/setup-ubuntu.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh # Required by lib hdfs. ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh - sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh + sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh ensure_pattern_matched 'run_and_time install_conda' scripts/setup-ubuntu.sh sed -i '/run_and_time install_conda/d' scripts/setup-ubuntu.sh # Just depends on Gluten to install arrow libs since Gluten will apply some patches to Arrow source and uses different build options. @@ -107,11 +104,10 @@ function process_setup_centos9 { ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh - sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh - sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh + sed -i '/^.*dnf_install autoconf/a\ dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh - sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh + sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh diff --git a/ep/build-velox/src/setup-centos7.sh b/ep/build-velox/src/setup-centos7.sh index 6f76112ae6a4..34d7bcfb65c5 100755 --- a/ep/build-velox/src/setup-centos7.sh +++ b/ep/build-velox/src/setup-centos7.sh @@ -74,8 +74,8 @@ function install_ninja { function install_folly { cd "${DEPENDENCY_DIR}" - github_checkout facebook/folly "${FB_OS_VERSION}" - cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON + wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly + cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } function install_conda { @@ -99,22 +99,19 @@ function install_openssl { function install_gflags { cd "${DEPENDENCY_DIR}" wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags - cd gflags - cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local + cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local } function install_glog { cd "${DEPENDENCY_DIR}" wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog - cd glog - cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr/local + cmake_install glog -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr/local } function install_snappy { cd "${DEPENDENCY_DIR}" wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy - cd snappy - cmake_install -DSNAPPY_BUILD_TESTS=OFF + cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF } function install_dwarf { @@ -213,10 +210,7 @@ function install_duckdb { if $BUILD_DUCKDB ; then echo 'Building DuckDB' wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - ( - cd duckdb - cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - ) + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release fi } diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index c8078893ae68..771c2ab835df 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -41,7 +41,10 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}" export CC=/opt/rh/gcc-toolset-9/root/bin/gcc export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ -DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} +FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" + function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" } @@ -74,27 +77,22 @@ function install_conda { function install_openssl { wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl - cd openssl - ./config no-shared && make depend && make && sudo make install - cd .. + ( + cd openssl + ./config no-shared && make depend && make && sudo make install + ) } function install_gflags { # Remove an older version if present. dnf remove -y gflags wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags - ( - cd gflags - cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 - ) + cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 } function install_glog { wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog - ( - cd glog - cmake_install -DBUILD_SHARED_LIBS=ON - ) + cmake_install glog -DBUILD_SHARED_LIBS=ON } function install_lzo { @@ -108,7 +106,7 @@ function install_lzo { } function install_boost { - wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost ( cd boost ./bootstrap.sh --prefix=/usr/local @@ -118,18 +116,12 @@ function install_boost { function install_snappy { wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy - ( - cd snappy - cmake_install -DSNAPPY_BUILD_TESTS=OFF - ) + cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF } function install_fmt { - wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt - ( - cd fmt - cmake_install -DFMT_TEST=OFF - ) + wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt + cmake_install fmt -DFMT_TEST=OFF } function install_protobuf { @@ -143,56 +135,36 @@ function install_protobuf { ) } -FB_OS_VERSION="v2024.05.20.00" - function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz - ( - cd fizz/fizz - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install fizz/fizz -DBUILD_TESTS=OFF } function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly - ( - cd folly - cmake_install -DFOLLY_HAVE_INT128_T=ON - ) + cmake_install folly -DFOLLY_HAVE_INT128_T=ON } function install_wangle { wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle - ( - cd wangle/wangle - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install wangle/wangle -DBUILD_TESTS=OFF } function install_fbthrift { wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift - ( - cd fbthrift - cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF - ) + cmake_install fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF } function install_mvfst { wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst - ( - cd mvfst - cmake_install -DBUILD_TESTS=OFF - ) + cmake_install mvfst -DBUILD_TESTS=OFF } function install_duckdb { if $BUILD_DUCKDB ; then echo 'Building DuckDB' wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb - ( - cd duckdb - cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release - ) + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release fi }