diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 1ca5cce9c7ecf..e038d10daf4e6 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -166,7 +166,7 @@ jobs: - name: Install Dependencies run: | - source scripts/setup-ubuntu.sh && install_apt_deps + source scripts/setup-ubuntu.sh && install_apt_deps && install_velox_deps - name: Clear CCache Statistics run: | diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 63672ec7aa87a..2e3baba920d8f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -66,8 +66,7 @@ jobs: run: | source scripts/setup-macos.sh install_build_prerequisites - install_velox_deps_from_brew - install_double_conversion + install_velox_deps echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV brew unlink protobuf || echo "protobuf not installed" diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh index 2b38fd1d4c733..c1fddd892d8a2 100755 --- a/scripts/setup-centos9.sh +++ b/scripts/setup-centos9.sh @@ -209,6 +209,11 @@ function install_cuda { dnf install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-') } +function install_proxygen { + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen + cmake_install_dir proxygen -DBUILD_TESTS=OFF +} + function install_velox_deps { run_and_time install_velox_deps_from_dnf run_and_time install_conda @@ -226,6 +231,7 @@ function install_velox_deps { run_and_time install_fbthrift run_and_time install_duckdb run_and_time install_arrow + run_and_time install_proxygen } (return 2> /dev/null) && return # If script was sourced, don't run commands. diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index 8d8863586df73..68465115d7ad1 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -39,10 +39,29 @@ export OS_CXXFLAGS=" -isystem $(brew --prefix)/include " NPROC=$(getconf _NPROCESSORS_ONLN) DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)} -MACOS_VELOX_DEPS="bison flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 snappy xz zstd" +MACOS_VELOX_DEPS="bison boost flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 snappy xz zstd" MACOS_BUILD_DEPS="ninja cmake" FB_OS_VERSION="v2024.05.20.00" FMT_VERSION="10.1.1" +XSIMD_VERSION="10.0.0" +ARROW_VERSION="15.0.0" +DUCKDB_VERSION="v0.8.1" +SIMDJSON_VERSION="3.9.3" +PROTOBUF_VERSION="21.8" + +# Adapter related versions. +ABSEIL_VERSION="20240116.2" +GRPC_VERSION="v1.48.1" +CRC32_VERSION="1.1.2" +NLOHMAN_JSON_VERSION="v3.11.3" +GOOGLE_CLOUD_CPP_VERSION="v2.22.0" +HADOOP_VERSION="2.10.1" +AZURE_SDK_VERSION="12.8.0" + + +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +USE_CLANG="${USE_CLANG:-false}" function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew @@ -96,16 +115,66 @@ function install_velox_deps_from_brew { done } +function install_xsimd { + wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd + cmake_install_dir xsimd +} + function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt cmake_install_dir fmt -DFMT_TEST=OFF } +function install_arrow { + set -e # Exit on first error + echo "Downloading and extracting Apache Arrow version ${ARROW_VERSION}..." + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + + echo "Starting the build process..." + ( + cd arrow/cpp + cmake_install_dir \ + -DARROW_PARQUET=OFF \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_RUNTIME_SIMD_LEVEL=NONE \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_TESTING=ON \ + -Dcmake_install_dir_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DARROW_BUILD_STATIC=ON \ + -DThrift_SOURCE=BUNDLED + + echo "Installing Thrift..." + cd _build/thrift_ep-prefix/src/thrift_ep-build + ${SUDO} cmake --install ./ --prefix /usr/local/ + ) + echo "Apache Arrow installation complete." +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb + cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + fi +} + function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf + cmake_install_dir protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package +} + function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF @@ -141,17 +210,79 @@ function install_re2 { cmake_install_dir re2 -DRE2_BUILD_TESTING=OFF } +function install_proxygen { + wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen + cmake_install_dir proxygen -DBUILD_TESTS=OFF +} + +function install_simdjson { + wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson + cmake_install_dir simdjson +} + +function install_gcs { + # Install gcs dependencies + # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries + + # abseil-cpp + github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1 + cmake_install_dir \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON + + # protobuf + github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1 + cmake_install_dir \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package + + # grpc + github_checkout grpc/grpc ${GRPC_VERSION} --depth 1 + cmake_install_dir \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON + + # crc32 + github_checkout google/crc32c ${CRC32_VERSION} --depth 1 + cmake_install_dir \ + -DCRC32C_BUILD_TESTS=OFF \ + -DCRC32C_BUILD_BENCHMARKS=OFF \ + -DCRC32C_USE_GLOG=OFF + + # nlohmann json + github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1 + cmake_install_dir \ + -DJSON_BuildTests=OFF + + # google-cloud-cpp + github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1 + cmake_install_dir \ + -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \ + -DGOOGLE_CLOUD_CPP_ENABLE=storage +} + function install_velox_deps { run_and_time install_velox_deps_from_brew - run_and_time install_ranges_v3 run_and_time install_double_conversion - run_and_time install_re2 run_and_time install_fmt run_and_time install_folly run_and_time install_fizz run_and_time install_wangle run_and_time install_mvfst run_and_time install_fbthrift + run_and_time install_xsimd + run_and_time install_proxygen + run_and_time install_arrow + run_and_time install_duckdb + run_and_time install_simdjson } (return 2> /dev/null) && return # If script was sourced, don't run commands.