Skip to content

Commit

Permalink
Adapt setup scripts for the changes in upstream velox
Browse files Browse the repository at this point in the history
  • Loading branch information
@liujiayi771
liujiayi771 committed Jul 28, 2024
1 parent bbb2e2f commit 2f45939
Show file tree
Hide file tree
Showing 3 changed files with 37 additions and 75 deletions.
26 changes: 11 additions & 15 deletions ep/build-velox/src/get_velox.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,25 +70,22 @@ function process_setup_ubuntu {
ensure_pattern_matched '\${SUDO} apt install -y libunwind-dev' scripts/setup-ubuntu.sh
sed -i 's/${SUDO} apt install -y libunwind-dev//' scripts/setup-ubuntu.sh
ensure_pattern_matched 'ccache' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ curl \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libiberty-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libxml2-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libkrb5-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libgsasl7-dev \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ libuuid1 \\' scripts/setup-ubuntu.sh
sed -i '/ccache/a\ uuid-dev \\' scripts/setup-ubuntu.sh
ensure_pattern_matched 'libgmock-dev' scripts/setup-ubuntu.sh
sed -i '/libgmock-dev/d' scripts/setup-ubuntu.sh # resolved by ep/build-velox/build/velox_ep/CMake/resolve_dependency_modules/gtest.cmake
ensure_pattern_matched 'github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive' scripts/setup-ubuntu.sh
sed -i 's/github_checkout boostorg\/boost \"\${BOOST_VERSION}\" --recursive/wget_and_untar https:\/\/github.com\/boostorg\/boost\/releases\/download\/boost-1.84.0\/boost-1.84.0.tar.gz boost \&\& cd boost/g' scripts/setup-ubuntu.sh
ensure_pattern_matched 'function install_folly' scripts/setup-ubuntu.sh
sed -i '/^function install_folly.*/i function install_protobuf {\n wget https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz\n tar -xzf protobuf-all-21.4.tar.gz\n cd protobuf-21.4\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n}\n' scripts/setup-ubuntu.sh
sed -i '/^function install_folly.*/i function install_protobuf {\n wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v21.4/protobuf-all-21.4.tar.gz protobuf\n (\n cd protobuf\n ./configure CXXFLAGS="-fPIC" --prefix=/usr/local\n make "-j$(nproc)"\n sudo make install\n sudo ldconfig\n )\n}\n' scripts/setup-ubuntu.sh
ensure_pattern_matched ' run_and_time install_folly' scripts/setup-ubuntu.sh
sed -i '/^ run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh
# Required by lib hdfs.
ensure_pattern_matched 'ccache ' scripts/setup-ubuntu.sh
sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
sed -i '/ccache /a\ yasm \\' scripts/setup-ubuntu.sh
ensure_pattern_matched 'run_and_time install_conda' scripts/setup-ubuntu.sh
sed -i '/run_and_time install_conda/d' scripts/setup-ubuntu.sh
# Just depends on Gluten to install arrow libs since Gluten will apply some patches to Arrow source and uses different build options.
Expand All @@ -107,11 +104,10 @@ function process_setup_centos9 {

ensure_pattern_matched 'dnf_install' scripts/setup-centos9.sh
sed -i 's/dnf_install ninja-build cmake curl ccache gcc-toolset-12 git/dnf_install ninja-build cmake curl ccache gcc-toolset-12/' scripts/setup-centos9.sh
sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos9.sh
sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh
sed -i '/^.*dnf_install autoconf/a\ dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos9.sh

ensure_pattern_matched 'install_gflags' scripts/setup-centos9.sh
sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh
sed -i '/^function install_gflags.*/i function install_openssl {\n wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl \n cd openssl \n ./config no-shared && make depend && make && sudo make install \n cd ..\n}\n' scripts/setup-centos9.sh

ensure_pattern_matched 'install_fbthrift' scripts/setup-centos9.sh
sed -i '/^ run_and_time install_fbthrift/a \ run_and_time install_openssl' scripts/setup-centos9.sh
Expand Down
18 changes: 6 additions & 12 deletions ep/build-velox/src/setup-centos7.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,8 +74,8 @@ function install_ninja {

function install_folly {
cd "${DEPENDENCY_DIR}"
github_checkout facebook/folly "${FB_OS_VERSION}"
cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
}

function install_conda {
Expand All @@ -99,22 +99,19 @@ function install_openssl {
function install_gflags {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
cd gflags
cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64 -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
}

function install_glog {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/google/glog/archive/v0.5.0.tar.gz glog
cd glog
cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
cmake_install glog -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DCMAKE_INSTALL_PREFIX:PATH=/usr/local
}

function install_snappy {
cd "${DEPENDENCY_DIR}"
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
cd snappy
cmake_install -DSNAPPY_BUILD_TESTS=OFF
cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
}

function install_dwarf {
Expand Down Expand Up @@ -213,10 +210,7 @@ function install_duckdb {
if $BUILD_DUCKDB ; then
echo 'Building DuckDB'
wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
(
cd duckdb
cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
)
cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
fi
}

Expand Down
68 changes: 20 additions & 48 deletions ep/build-velox/src/setup-centos8.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,10 @@ BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
export CXX=/opt/rh/gcc-toolset-9/root/bin/g++

DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
FB_OS_VERSION="v2024.05.20.00"
FMT_VERSION="10.1.1"
BOOST_VERSION="boost-1.84.0"

function dnf_install {
dnf install -y -q --setopt=install_weak_deps=False "$@"
}
Expand Down Expand Up @@ -74,27 +77,22 @@ function install_conda {

function install_openssl {
wget_and_untar https://github.com/openssl/openssl/archive/refs/tags/OpenSSL_1_1_1s.tar.gz openssl
cd openssl
./config no-shared && make depend && make && sudo make install
cd ..
(
cd openssl
./config no-shared && make depend && make && sudo make install
)
}

function install_gflags {
# Remove an older version if present.
dnf remove -y gflags
wget_and_untar https://github.com/gflags/gflags/archive/v2.2.2.tar.gz gflags
(
cd gflags
cmake_install -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
)
cmake_install gflags -DBUILD_SHARED_LIBS=ON -DBUILD_STATIC_LIBS=ON -DBUILD_gflags_LIB=ON -DLIB_SUFFIX=64
}

function install_glog {
wget_and_untar https://github.com/google/glog/archive/v0.6.0.tar.gz glog
(
cd glog
cmake_install -DBUILD_SHARED_LIBS=ON
)
cmake_install glog -DBUILD_SHARED_LIBS=ON
}

function install_lzo {
Expand All @@ -108,7 +106,7 @@ function install_lzo {
}

function install_boost {
wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost
(
cd boost
./bootstrap.sh --prefix=/usr/local
Expand All @@ -118,18 +116,12 @@ function install_boost {

function install_snappy {
wget_and_untar https://github.com/google/snappy/archive/1.1.8.tar.gz snappy
(
cd snappy
cmake_install -DSNAPPY_BUILD_TESTS=OFF
)
cmake_install snappy -DSNAPPY_BUILD_TESTS=OFF
}

function install_fmt {
wget_and_untar https://github.com/fmtlib/fmt/archive/10.1.1.tar.gz fmt
(
cd fmt
cmake_install -DFMT_TEST=OFF
)
wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
cmake_install fmt -DFMT_TEST=OFF
}

function install_protobuf {
Expand All @@ -143,56 +135,36 @@ function install_protobuf {
)
}

FB_OS_VERSION="v2024.05.20.00"

function install_fizz {
wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
(
cd fizz/fizz
cmake_install -DBUILD_TESTS=OFF
)
cmake_install fizz/fizz -DBUILD_TESTS=OFF
}

function install_folly {
wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
(
cd folly
cmake_install -DFOLLY_HAVE_INT128_T=ON
)
cmake_install folly -DFOLLY_HAVE_INT128_T=ON
}

function install_wangle {
wget_and_untar https://github.com/facebook/wangle/archive/refs/tags/${FB_OS_VERSION}.tar.gz wangle
(
cd wangle/wangle
cmake_install -DBUILD_TESTS=OFF
)
cmake_install wangle/wangle -DBUILD_TESTS=OFF
}

function install_fbthrift {
wget_and_untar https://github.com/facebook/fbthrift/archive/refs/tags/${FB_OS_VERSION}.tar.gz fbthrift
(
cd fbthrift
cmake_install -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
)
cmake_install fbthrift -Denable_tests=OFF -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF
}

function install_mvfst {
wget_and_untar https://github.com/facebook/mvfst/archive/refs/tags/${FB_OS_VERSION}.tar.gz mvfst
(
cd mvfst
cmake_install -DBUILD_TESTS=OFF
)
cmake_install mvfst -DBUILD_TESTS=OFF
}

function install_duckdb {
if $BUILD_DUCKDB ; then
echo 'Building DuckDB'
wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/v0.8.1.tar.gz duckdb
(
cd duckdb
cmake_install -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
)
cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=Release
fi
}

Expand Down

0 comments on commit 2f45939

Please sign in to comment.