From 01ec015f1ff765b6949074f1d677d2a3ab5dc94a Mon Sep 17 00:00:00 2001 From: Joe Abraham Date: Fri, 20 Sep 2024 10:48:42 +0530 Subject: [PATCH] [Temp] Needs to be removed --- .github/workflows/macos.yml | 10 +-- scripts/setup-macos.sh | 129 +++++++++++++++++++++++++++++++++++- 2 files changed, 128 insertions(+), 11 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 91f3b8f6613e2..2e3baba920d8f 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -66,15 +66,7 @@ jobs: run: | source scripts/setup-macos.sh install_build_prerequisites - install_velox_deps_from_brew - install_double_conversion - install_fmt - install_folly - install_fizz - install_wangle - install_mvfst - install_fbthrift - install_proxygen + install_velox_deps echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV brew unlink protobuf || echo "protobuf not installed" diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh index da3442861ad52..68465115d7ad1 100755 --- a/scripts/setup-macos.sh +++ b/scripts/setup-macos.sh @@ -43,6 +43,25 @@ MACOS_VELOX_DEPS="bison boost flex gflags glog googletest icu4c libevent libsodi MACOS_BUILD_DEPS="ninja cmake" FB_OS_VERSION="v2024.05.20.00" FMT_VERSION="10.1.1" +XSIMD_VERSION="10.0.0" +ARROW_VERSION="15.0.0" +DUCKDB_VERSION="v0.8.1" +SIMDJSON_VERSION="3.9.3" +PROTOBUF_VERSION="21.8" + +# Adapter related versions. +ABSEIL_VERSION="20240116.2" +GRPC_VERSION="v1.48.1" +CRC32_VERSION="1.1.2" +NLOHMAN_JSON_VERSION="v3.11.3" +GOOGLE_CLOUD_CPP_VERSION="v2.22.0" +HADOOP_VERSION="2.10.1" +AZURE_SDK_VERSION="12.8.0" + + +CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" +BUILD_DUCKDB="${BUILD_DUCKDB:-true}" +USE_CLANG="${USE_CLANG:-false}" function update_brew { DEFAULT_BREW_PATH=/usr/local/bin/brew @@ -96,16 +115,66 @@ function install_velox_deps_from_brew { done } +function install_xsimd { + wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd + cmake_install_dir xsimd +} + function install_fmt { wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt cmake_install_dir fmt -DFMT_TEST=OFF } +function install_arrow { + set -e # Exit on first error + echo "Downloading and extracting Apache Arrow version ${ARROW_VERSION}..." + wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow + + echo "Starting the build process..." + ( + cd arrow/cpp + cmake_install_dir \ + -DARROW_PARQUET=OFF \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_SIMD_LEVEL=NONE \ + -DARROW_RUNTIME_SIMD_LEVEL=NONE \ + -DARROW_WITH_UTF8PROC=OFF \ + -DARROW_TESTING=ON \ + -Dcmake_install_dir_PREFIX=/usr/local \ + -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \ + -DARROW_BUILD_STATIC=ON \ + -DThrift_SOURCE=BUNDLED + + echo "Installing Thrift..." + cd _build/thrift_ep-prefix/src/thrift_ep-build + ${SUDO} cmake --install ./ --prefix /usr/local/ + ) + echo "Apache Arrow installation complete." +} + +function install_duckdb { + if $BUILD_DUCKDB ; then + echo 'Building DuckDB' + wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb + cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} + fi +} + function install_folly { wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON } +function install_protobuf { + wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf + cmake_install_dir protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package +} + function install_fizz { wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF @@ -146,18 +215,74 @@ function install_proxygen { cmake_install_dir proxygen -DBUILD_TESTS=OFF } +function install_simdjson { + wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson + cmake_install_dir simdjson +} + +function install_gcs { + # Install gcs dependencies + # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries + + # abseil-cpp + github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1 + cmake_install_dir \ + -DABSL_BUILD_TESTING=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DABSL_PROPAGATE_CXX_STD=ON \ + -DABSL_ENABLE_INSTALL=ON + + # protobuf + github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1 + cmake_install_dir \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_ABSL_PROVIDER=package + + # grpc + github_checkout grpc/grpc ${GRPC_VERSION} --depth 1 + cmake_install_dir \ + -DgRPC_BUILD_TESTS=OFF \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_INSTALL=ON + + # crc32 + github_checkout google/crc32c ${CRC32_VERSION} --depth 1 + cmake_install_dir \ + -DCRC32C_BUILD_TESTS=OFF \ + -DCRC32C_BUILD_BENCHMARKS=OFF \ + -DCRC32C_USE_GLOG=OFF + + # nlohmann json + github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1 + cmake_install_dir \ + -DJSON_BuildTests=OFF + + # google-cloud-cpp + github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1 + cmake_install_dir \ + -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \ + -DGOOGLE_CLOUD_CPP_ENABLE=storage +} + function install_velox_deps { run_and_time install_velox_deps_from_brew - run_and_time install_ranges_v3 run_and_time install_double_conversion - run_and_time install_re2 run_and_time install_fmt run_and_time install_folly run_and_time install_fizz run_and_time install_wangle run_and_time install_mvfst run_and_time install_fbthrift + run_and_time install_xsimd run_and_time install_proxygen + run_and_time install_arrow + run_and_time install_duckdb + run_and_time install_simdjson } (return 2> /dev/null) && return # If script was sourced, don't run commands.