diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index d5e33e926d82..4e0882a834dc 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -13,7 +13,7 @@ BUILD_TESTS=OFF BUILD_EXAMPLES=OFF BUILD_BENCHMARKS=OFF BUILD_JEMALLOC=OFF -BUILD_PROTOBUF=ON +BUILD_PROTOBUF=OFF BUILD_VELOX_TESTS=OFF BUILD_VELOX_BENCHMARKS=OFF ENABLE_QAT=OFF @@ -201,7 +201,7 @@ function build_arrow { function build_velox { echo "Start to build Velox" cd $GLUTEN_DIR/ep/build-velox/src - ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER + ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS $VELOX_PARAMETER # When BUILD_TESTS is on for gluten cpp, we need turn on VELOX_BUILD_TEST_UTILS via build_test_utils. ./build_velox.sh --run_setup_script=$RUN_SETUP_SCRIPT --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_test_utils=$BUILD_TESTS --build_tests=$BUILD_VELOX_TESTS --build_benchmarks=$BUILD_VELOX_BENCHMARKS \ diff --git a/docs/get-started/build-guide.md b/docs/get-started/build-guide.md index dc4989bc8642..a9e9bd1266df 100644 --- a/docs/get-started/build-guide.md +++ b/docs/get-started/build-guide.md @@ -14,8 +14,8 @@ Please set them via `--`, e.g. `--build_type=Release`. | build_tests | Build gluten cpp tests. | OFF | | build_examples | Build udf example. | OFF | | build_benchmarks | Build gluten cpp benchmarks. | OFF | -| build_jemalloc | Build with jemalloc. | OFF | -| build_protobuf | Build protobuf lib. | ON | +| build_jemalloc | Build with jemalloc. | OFF | +| build_protobuf | Build protobuf lib. | OFF | | enable_qat | Enable QAT for shuffle data de/compression. | OFF | | enable_iaa | Enable IAA for shuffle data de/compression. | OFF | | enable_hbm | Enable HBM allocator. | OFF | diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 0adc1ce8ff61..d33e684e931a 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -22,8 +22,6 @@ VELOX_HOME="" #Set on run gluten on HDFS ENABLE_HDFS=OFF -#It can be set to OFF when compiling velox again -BUILD_PROTOBUF=ON #Set on run gluten on S3 ENABLE_S3=OFF #Set on run gluten on GCS @@ -47,10 +45,6 @@ for arg in "$@"; do VELOX_HOME=("${arg#*=}") shift # Remove argument name from processing ;; - --build_protobuf=*) - BUILD_PROTOBUF=("${arg#*=}") - shift # Remove argument name from processing - ;; --enable_hdfs=*) ENABLE_HDFS=("${arg#*=}") shift # Remove argument name from processing @@ -95,15 +89,13 @@ function process_setup_ubuntu { sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive/wget_and_untar https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz boost \&\& cd boost/g' scripts/setup-ubuntu.sh + sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh + sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_folly.*/i function install_libhdfs3 {\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-ubuntu.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh fi - if [ $BUILD_PROTOBUF == "ON" ]; then - sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh - fi sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-ubuntu.sh @@ -136,15 +128,14 @@ function process_setup_centos8 { sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos8.sh sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos8.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos8.sh + sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh if [ $ENABLE_HDFS == "ON" ]; then sed -i '/^function install_gflags.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master\n cmake_install\n}\n' scripts/setup-centos8.sh sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_libhdfs3' scripts/setup-centos8.sh sed -i '/^ dnf_install ninja-build/a\ dnf_install yasm\' scripts/setup-centos8.sh fi - if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then - sed -i '/cd protobuf/{n;s/\.\/configure --prefix=\/usr/\.\/configure CXXFLAGS="-fPIC" --prefix=\/usr\/local/;}' scripts/setup-centos8.sh - fi + sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_fbthrift/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos8.sh @@ -172,15 +163,12 @@ function process_setup_centos7 { # install gtest sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh - + sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh if [ $ENABLE_HDFS = "ON" ]; then sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh fi - if [[ $BUILD_PROTOBUF == "ON" ]] || [[ $ENABLE_HDFS == "ON" ]]; then - sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh - fi sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh if [ $ENABLE_S3 == "ON" ]; then sed -i '/^ run_and_time install_folly/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh aws' scripts/setup-centos7.sh @@ -219,7 +207,6 @@ function process_setup_tencentos32 { echo "Preparing Velox source code..." echo "ENABLE_HDFS=${ENABLE_HDFS}" -echo "BUILD_PROTOBUF=${BUILD_PROTOBUF}" CURRENT_DIR=$( cd "$(dirname "$BASH_SOURCE")"