From b1be2c321278a4ef39881705dbcce3d5dbf6ac0c Mon Sep 17 00:00:00 2001 From: Joe Abraham Date: Tue, 3 Sep 2024 09:06:06 +0530 Subject: [PATCH] add proxygen --- scripts/setup-macos.sh | 63 ++++++++++++++++++++------- velox/functions/remote/CMakeLists.txt | 12 +++++ 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 450324e8b19af..288063a19e518 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -14,7 +14,7 @@ # limitations under the License. # This script documents setting up a macOS host for Velox -# development. Running it should make you ready to compile. +# development. Running it should make you ready to compile. # # Environment variables: # * INSTALL_PREREQUISITES="N": Skip installation of brew/pip deps. @@ -35,16 +35,18 @@ PYTHON_VENV=${PYHTON_VENV:-"${SCRIPTDIR}/../.venv"} NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -MACOS_VELOX_DEPS="bison boost double-conversion flex fmt gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz xsimd zstd" -MACOS_BUILD_DEPS="ninja cmake ccache" +MACOS_VELOX_DEPS="bison double-conversion flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz zstd" +MACOS_BUILD_DEPS="ninja cmake" FB_OS_VERSION="v2024.05.20.00" FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +XSIMD_VERSION="10.0.0" + function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew - if [ `arch` == "arm64" ] ; - then - DEFAULT_BREW_PATH=$(which brew) ; + if [ "$(arch)" == "arm64" ]; then + DEFAULT_BREW_PATH=$(which brew) fi BREW_PATH=${BREW_PATH:-$DEFAULT_BREW_PATH} $BREW_PATH update --auto-update --verbose @@ -53,44 +55,72 @@ function update_brew { function install_from_brew { pkg=$1 - if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]]; - then + if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]]; then pkg=${BASH_REMATCH[1]} ver=${BASH_REMATCH[2]} echo "Installing '${pkg}' at '${ver}'" tap="velox/local-${pkg}" brew tap-new "${tap}" brew extract "--version=${ver}" "${pkg}" "${tap}" - brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 ) + brew install "${tap}/${pkg}@${ver}" || { echo "Failed to install ${tap}/${pkg}@${ver}"; exit 1; } else - ( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 ) + (brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful") || brew upgrade --formula "${pkg}" || { echo "Failed to install ${pkg}"; exit 1; } fi } + function install_build_prerequisites { - for pkg in ${MACOS_BUILD_DEPS} - do + for pkg in ${MACOS_BUILD_DEPS}; do install_from_brew ${pkg} done + if [ ! -f ${PYTHON_VENV}/pyvenv.cfg ]; then echo "Creating Python Virtual Environment at ${PYTHON_VENV}" python3 -m venv ${PYTHON_VENV} fi - source ${PYTHON_VENV}/bin/activate; pip3 install cmake-format regex pyyaml + source ${PYTHON_VENV}/bin/activate + pip3 install cmake-format regex pyyaml } function install_velox_deps_from_brew { - for pkg in ${MACOS_VELOX_DEPS} - do + for pkg in ${MACOS_VELOX_DEPS}; do install_from_brew ${pkg} done } +function install_xsimd { + wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd + cmake_install xsimd +} + function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt cmake_install fmt -DFMT_TEST=OFF } +function install_proxygen { + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen + cmake_install proxygen -DBUILD_TESTS=OFF +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost + ( + cd boost + if [[ ${USE_CLANG} != "false" ]]; then + ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15" + # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to + # clang of version 15 when toolset clang-15 is used. + # This reconciles the project-config.jam generation with what the b2 build system allows for customization. + sed -i 's/using clang-15/using clang : 15/g' project-config.jam + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi toolset=clang-15 --without-python + else + ./bootstrap.sh --prefix=/usr/local + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python + fi + ) +} + function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly cmake_install folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON @@ -142,6 +172,9 @@ function install_velox_deps { run_and_time install_wangle run_and_time install_mvfst run_and_time install_fbthrift + run_and_time install_boost + run_and_time install_xsimd + run_and_time install_proxygen } (return 2> /dev/null) && return # If script was sourced, don't run commands. diff --git a/velox/functions/remote/CMakeLists.txt b/velox/functions/remote/CMakeLists.txt index a38c65894a2ff..c5f32ca662cb4 100644 --- a/velox/functions/remote/CMakeLists.txt +++ b/velox/functions/remote/CMakeLists.txt @@ -14,10 +14,22 @@ if(NOT DEFINED PROXYGEN_LIBRARIES) find_package(Sodium REQUIRED) + find_library(PROXYGEN proxygen) find_library(PROXYGEN_HTTP_SERVER proxygenhttpserver) find_library(FIZZ fizz) find_library(WANGLE wangle) + + if(NOT PROXYGEN + OR NOT PROXYGEN_HTTP_SERVER + OR NOT FIZZ + OR NOT WANGLE) + message( + FATAL_ERROR + "One or more proxygen libraries were not found. Please ensure proxygen, proxygenhttpserver, fizz, and wangle are installed." + ) + endif() + set(PROXYGEN_LIBRARIES ${PROXYGEN_HTTP_SERVER} ${PROXYGEN} ${WANGLE} ${FIZZ}) endif()