diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 797fc87be4196..2afab60a0e0cd 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -63,9 +63,7 @@ jobs: env: HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK: "TRUE" run: | - source scripts/setup-macos.sh - install_build_prerequisites - install_velox_deps_from_brew + scripts/setup-macos.sh echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV brew unlink protobuf || echo "protobuf not installed" diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index fd5e61012cda2..85802147273a2 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -28,6 +28,36 @@ set -e # Exit on error. set -x # Print commands that are executed. + +# Build dependencies versions. +FB_OS_VERSION="v2024.05.20.00" +FMT_VERSION="10.1.1" +BOOST_VERSION="boost-1.84.0" +ARROW_VERSION="15.0.0" +DUCKDB_VERSION="v0.8.1" +PROTOBUF_VERSION="21.8" +MINIO_VERSION="2022-05-26T05-48-41Z" +MINIO_BINARY_NAME="minio-2022-05-26" +AWS_SDK_VERSION="1.11.321" +XSIMD_VERSION="10.0.0" +SIMDJSON_VERSION="3.9.3" +CPR_VERSION="1.10.5" +DOUBLE_CONVERSION_VERSION="v3.1.5" +RANGE_V3_VERSION="0.12.0" +RE2_VERSION="2022-02-01" +GLOG_VERSION="v0.6.0" +LZO_VERSION="2.10" +SNAPPY_VERSION="1.1.8" + +# Adapter related versions. +ABSEIL_VERSION="20240116.2" +GRPC_VERSION="v1.48.1" +CRC32_VERSION="1.1.2" +NLOHMAN_JSON_VERSION="v3.11.3" +GOOGLE_CLOUD_CPP_VERSION="v2.22.0" +HADOOP_VERSION="2.10.1" +AZURE_SDK_VERSION="12.8.0" + SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") source $SCRIPTDIR/setup-helper-functions.sh PYTHON_VENV=${PYHTON_VENV:-"${SCRIPTDIR}/../.venv"} @@ -37,14 +67,16 @@ NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} MACOS_VELOX_DEPS="bison boost double-conversion flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 simdjson snappy thrift xz xsimd zstd" MACOS_BUILD_DEPS="ninja cmake" -FB_OS_VERSION="v2024.05.20.00" -FMT_VERSION="10.1.1" + +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +USE_CLANG="${USE_CLANG:-false}" function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew - if [ `arch` == "arm64" ] ; + if [ "$(arch)" == "arm64" ]; then - DEFAULT_BREW_PATH=$(which brew) ; + DEFAULT_BREW_PATH=$(which brew) fi BREW_PATH=${BREW_PATH:-$DEFAULT_BREW_PATH} $BREW_PATH update --auto-update --verbose @@ -61,9 +93,9 @@ function install_from_brew { tap="velox/local-${pkg}" brew tap-new "${tap}" brew extract "--version=${ver}" "${pkg}" "${tap}" - brew install "${tap}/${pkg}@${ver}" || ( echo "Failed to install ${tap}/${pkg}@${ver}" ; exit 1 ) + brew install "${tap}/${pkg}@${ver}" || { echo "Failed to install ${tap}/${pkg}@${ver}"; exit 1; } else - ( brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful" || brew upgrade --formula "$pkg" ) || ( echo "Failed to install ${pkg}" ; exit 1 ) + (brew install --formula "${pkg}" && echo "Installation of ${pkg} is successful") || brew upgrade --formula "${pkg}" || { echo "Failed to install ${pkg}"; exit 1; } fi } @@ -83,7 +115,7 @@ function install_build_prerequisites { } function install_velox_deps_from_brew { - for pkg in ${MACOS_VELOX_DEPS} + for pkg in ${MACOS_VELOX_DEPS}; do install_from_brew ${pkg} done @@ -134,6 +166,133 @@ function install_re2 { cmake_install re2 -DRE2_BUILD_TESTING=OFF } +function install_xsimd { + wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd + cmake_install xsimd +} + +function install_arrow { + set -e # Exit on first error + echo "Downloading and extracting Apache Arrow version ${ARROW_VERSION}..." + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + + echo "Starting the build process..." + ( + cd arrow/cpp + cmake_install \ + -DARROW_PARQUET=OFF \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_RUNTIME_SIMD_LEVEL=NONE \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_TESTING=ON \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DARROW_BUILD_STATIC=ON \ + -DThrift_SOURCE=BUNDLED + + echo "Installing Thrift..." + cd _build/thrift_ep-prefix/src/thrift_ep-build + ${SUDO} cmake --install ./ --prefix /usr/local/ + ) + echo "Apache Arrow installation complete." +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb + cmake_install duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + fi +} + +function install_boost { + wget_and_untar https://github.com/boostorg/boost/releases/download/${BOOST_VERSION}/${BOOST_VERSION}.tar.gz boost + ( + cd boost + if [[ ${USE_CLANG} != "false" ]]; then + ./bootstrap.sh --prefix=/usr/local --with-toolset="clang-15" + # Switch the compiler from the clang-15 toolset which doesn't exist (clang-15.jam) to + # clang of version 15 when toolset clang-15 is used. + # This reconciles the project-config.jam generation with what the b2 build system allows for customization. + sed -i 's/using clang-15/using clang : 15/g' project-config.jam + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi toolset=clang-15 --without-python + else + ./bootstrap.sh --prefix=/usr/local + ${SUDO} ./b2 "-j${NPROC}" -d0 install threading=multi --without-python + fi + ) +} + +function install_proxygen { + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen + cmake_install proxygen -DBUILD_TESTS=OFF +} + +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf + cmake_install protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package +} + +function install_simdjson { + wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson + cmake_install simdjson +} + +function install_gcs { + # Install gcs dependencies + # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries + + # abseil-cpp + github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1 + cmake_install \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON + + # protobuf + github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1 + cmake_install \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package + + # grpc + github_checkout grpc/grpc ${GRPC_VERSION} --depth 1 + cmake_install \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON + + # crc32 + github_checkout google/crc32c ${CRC32_VERSION} --depth 1 + cmake_install \ + -DCRC32C_BUILD_TESTS=OFF \ + -DCRC32C_BUILD_BENCHMARKS=OFF \ + -DCRC32C_USE_GLOG=OFF + + # nlohmann json + github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1 + cmake_install \ + -DJSON_BuildTests=OFF + + # google-cloud-cpp + github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1 + cmake_install \ + -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \ + -DGOOGLE_CLOUD_CPP_ENABLE=storage +} + function install_velox_deps { run_and_time install_velox_deps_from_brew run_and_time install_ranges_v3 @@ -145,6 +304,13 @@ function install_velox_deps { run_and_time install_wangle run_and_time install_mvfst run_and_time install_fbthrift + run_and_time install_xsimd + run_and_time install_proxygen + run_and_time install_arrow + run_and_time install_duckdb + run_and_time install_protobuf + run_and_time install_simdjson + run_and_time install_simdjson } (return 2> /dev/null) && return # If script was sourced, don't run commands.