From 8a1d22556c8c5435218faed95227c7449550eafb Mon Sep 17 00:00:00 2001 From: Deepak Majeti Date: Fri, 29 Mar 2024 13:51:47 -0700 Subject: [PATCH] Add setup functions for velox and build dependencies handled by package managers (#8917) Summary: In the current setup scripts, the packages handled by the package manager are always installed in Centos8 and Ubuntu. This is unnecessary if we want to install a specific package. Make these installs optional by wrapping them around a function. Split the brew packages on MacOS based on the build vs Velox requirements. Pull Request resolved: https://github.com/facebookincubator/velox/pull/8917 Reviewed By: xiaoxmeng Differential Revision: D55506473 Pulled By: kgpai fbshipit-source-id: 3e4fc31080faac6006dede1808ac9d76cb4b5f60 --- scripts/setup-centos8.sh | 64 ++++++++++++++++------ scripts/setup-macos.sh | 57 ++++++++++++-------- scripts/setup-ubuntu.sh | 111 +++++++++++++++++++++++++-------------- 3 files changed, 155 insertions(+), 77 deletions(-) diff --git a/scripts/setup-centos8.sh b/scripts/setup-centos8.sh index 5903d1b9c35a..cf668b61d536 100755 --- a/scripts/setup-centos8.sh +++ b/scripts/setup-centos8.sh @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This script documents setting up a Centos8 host for Velox +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of packages for build. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ scripts/setup-centos8.sh install_googletest install_fmt +# + set -efx -o pipefail # Some of the packages must be build with the same compiler flags # so that some low level types are the same size. Also, disable warnings. @@ -25,27 +37,32 @@ export CXXFLAGS=$CFLAGS # Used by boost. export CPPFLAGS=$CFLAGS # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +export CC=/opt/rh/gcc-toolset-9/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ function dnf_install { dnf install -y -q --setopt=install_weak_deps=False "$@" } -dnf update -y -dnf_install epel-release dnf-plugins-core # For ccache, ninja -dnf config-manager --set-enabled powertools -dnf update -y -dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which libevent-devel \ - openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ - libdwarf-devel curl-devel libicu-devel - -dnf_install autoconf automake libtool bison flex python3 libsodium-devel +# Install packages required for build. +function install_build_prerequisites { + dnf update -y + dnf_install epel-release dnf-plugins-core # For ccache, ninja + dnf config-manager --set-enabled powertools + dnf update -y + dnf_install ninja-build cmake curl ccache gcc-toolset-9 git wget which + dnf_install autoconf automake python39 python39-devel python39-pip libtool +} -# install sphinx for doc gen -pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme +# Install dependencies from the package managers. +function install_velox_deps_from_dnf { + dnf_install libevent-devel \ + openssl-devel re2-devel libzstd-devel lz4-devel double-conversion-devel \ + libdwarf-devel curl-devel libicu-devel bison flex libsodium-devel -# Activate gcc9; enable errors on unset variables afterwards. -source /opt/rh/gcc-toolset-9/enable || exit 1 -set -u + # install sphinx for doc gen + pip3 install sphinx sphinx-tabs breathe sphinx_rtd_theme +} function install_conda { dnf_install conda @@ -169,6 +186,7 @@ function install_duckdb { } function install_velox_deps { + run_and_time install_velox_deps_from_dnf run_and_time install_conda run_and_time install_gflags run_and_time install_glog @@ -189,14 +207,26 @@ function install_velox_deps { ( if [[ $# -ne 0 ]]; then + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi + # Activate gcc9; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-9/enable || exit 1 + set -u install_velox_deps + echo "All dependencies for Velox installed!" + dnf clean all fi ) -echo "All dependencies for Velox installed!" - -dnf clean all diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 36252d5f0548..aa47e298b6a4 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -34,8 +34,8 @@ source $SCRIPTDIR/setup-helper-functions.sh NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -MACOS_DEPS="ninja flex bison cmake ccache protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium" - +MACOS_VELOX_DEPS="flex bison protobuf@21 icu4c boost gflags glog libevent lz4 lzo snappy xz zstd openssl libsodium" +MACOS_BUILD_DEPS="ninja cmake ccache" FB_OS_VERSION="v2024.02.26.00" function update_brew { @@ -49,26 +49,37 @@ function update_brew { $BREW_PATH developer off } +function install_from_brew { + pkg=$1 + if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]]; + then + pkg=${BASH_REMATCH[1]} + ver=${BASH_REMATCH[2]} + echo "Installing '${pkg}' at '${ver}'" + tap="velox/local-${pkg}" + brew tap-new "${tap}" + brew extract "--version=${ver}" "${pkg}" "${tap}" + brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 ) + else + ( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 ) + fi +} + function install_build_prerequisites { - for pkg in ${MACOS_DEPS} + for pkg in ${MACOS_BUILD_DEPS} do - if [[ "${pkg}" =~ ^([0-9a-z-]*):([0-9](\.[0-9\])*)$ ]]; - then - pkg=${BASH_REMATCH[1]} - ver=${BASH_REMATCH[2]} - echo "Installing '${pkg}' at '${ver}'" - tap="velox/local-${pkg}" - brew tap-new "${tap}" - brew extract "--version=${ver}" "${pkg}" "${tap}" - brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 ) - else - ( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 ) - fi + install_from_brew ${pkg} done - pip3 install --user cmake-format regex } +function install_velox_deps_from_brew { + for pkg in ${MACOS_VELOX_DEPS} + do + install_from_brew ${pkg} + done +} + function install_fmt { github_checkout fmtlib/fmt 10.1.1 cmake_install -DFMT_TEST=OFF @@ -116,9 +127,7 @@ function install_re2 { } function install_velox_deps { - if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then - run_and_time install_build_prerequisites - fi + run_and_time install_velox_deps_from_brew run_and_time install_ranges_v3 run_and_time install_double_conversion run_and_time install_re2 @@ -133,17 +142,23 @@ function install_velox_deps { (return 2> /dev/null) && return # If script was sourced, don't run commands. ( - echo "Installing mac dependencies" update_brew if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi install_velox_deps + echo "All deps for Velox installed! Now try \"make\"" fi ) -echo "All deps for Velox installed! Now try \"make\"" echo 'To add cmake-format bin to your $PATH, consider adding this to your ~/.profile:' echo 'export PATH=$HOME/bin:$HOME/Library/Python/3.7/bin:$PATH' diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh index 197cb6ba3562..d15e92c2a1ae 100755 --- a/scripts/setup-ubuntu.sh +++ b/scripts/setup-ubuntu.sh @@ -13,6 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This script documents setting up a Centos8 host for Velox +# development. Running it should make you ready to compile. +# +# Environment variables: +# * INSTALL_PREREQUISITES="N": Skip installation of packages for build. +# * PROMPT_ALWAYS_RESPOND="n": Automatically respond to interactive prompts. +# Use "n" to never wipe directories. +# +# You can also run individual functions below by specifying them as arguments: +# $ scripts/setup-ubuntu.sh install_googletest install_fmt +# + # Minimal setup for Ubuntu 20.04. set -eufx -o pipefail SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") @@ -31,43 +43,50 @@ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} export CMAKE_BUILD_TYPE=Release SUDO="${SUDO:-"sudo --preserve-env"}" -# Install all velox and folly dependencies. -# The is an issue on 22.04 where a version conflict prevents glog install, -# installing libunwind first fixes this. - -${SUDO} apt update -${SUDO} apt install -y libunwind-dev -${SUDO} apt install -y \ - g++ \ - cmake \ - ccache \ - ninja-build \ - checkinstall \ - git \ - libc-ares-dev \ - libcurl4-openssl-dev \ - libssl-dev \ - libicu-dev \ - libdouble-conversion-dev \ - libgoogle-glog-dev \ - libbz2-dev \ - libgflags-dev \ - libgmock-dev \ - libevent-dev \ - liblz4-dev \ - libzstd-dev \ - libre2-dev \ - libsnappy-dev \ - libsodium-dev \ - libthrift-dev \ - liblzo2-dev \ - libelf-dev \ - libdwarf-dev \ - bison \ - flex \ - libfl-dev \ - tzdata \ - wget +# Install packages required for build. +function install_build_prerequisites { + ${SUDO} apt update + # The is an issue on 22.04 where a version conflict prevents glog install, + # installing libunwind first fixes this. + ${SUDO} apt install -y libunwind-dev + ${SUDO} apt install -y \ + build-essential \ + cmake \ + ccache \ + ninja-build \ + checkinstall \ + git \ + wget +} + +# Install packages required for build. +function install_velox_deps_from_apt { + ${SUDO} apt update + ${SUDO} apt install -y \ + libc-ares-dev \ + libcurl4-openssl-dev \ + libssl-dev \ + libicu-dev \ + libdouble-conversion-dev \ + libgoogle-glog-dev \ + libbz2-dev \ + libgflags-dev \ + libgmock-dev \ + libevent-dev \ + liblz4-dev \ + libzstd-dev \ + libre2-dev \ + libsnappy-dev \ + libsodium-dev \ + libthrift-dev \ + liblzo2-dev \ + libelf-dev \ + libdwarf-dev \ + bison \ + flex \ + libfl-dev \ + tzdata +} function install_fmt { github_checkout fmtlib/fmt "${FMT_VERSION}" @@ -124,6 +143,7 @@ function install_conda { function install_velox_deps { + run_and_time install_velox_deps_from_apt run_and_time install_fmt run_and_time install_boost run_and_time install_folly @@ -134,16 +154,29 @@ function install_velox_deps { run_and_time install_conda } -(return 2> /dev/null) && return # If script was sourced, don't run commands. +function install_apt_deps { + install_build_prerequisites + install_velox_deps_from_apt +} + +# For backward compatibility, invoke install_apt_deps +(return 2> /dev/null) && install_apt_deps && return # If script was sourced, don't run commands. ( if [[ $# -ne 0 ]]; then for cmd in "$@"; do run_and_time "${cmd}" done + echo "All specified dependencies installed!" else + if [ "${INSTALL_PREREQUISITES:-Y}" == "Y" ]; then + echo "Installing build dependencies" + run_and_time install_build_prerequisites + else + echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" + fi install_velox_deps + echo "All dependencies for Velox installed!" fi ) -echo "All dependencies for Velox installed!"