[Temp] Needs to be removed

Joe-Abraham · Sep 20, 2024 · ce2ebbe · ce2ebbe
1 parent 0b096b9
commit ce2ebbe
Show file tree

Hide file tree

Showing 4 changed files with 142 additions and 6 deletions.
diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml
@@ -166,7 +166,7 @@ jobs:
 
       - name: Install Dependencies
         run: |
-          source scripts/setup-ubuntu.sh && install_apt_deps
+          source scripts/setup-ubuntu.sh && install_apt_deps  && install_velox_deps
 
       - name: Clear CCache Statistics
         run: |

diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -66,8 +66,7 @@ jobs:
         run: |
           source scripts/setup-macos.sh
           install_build_prerequisites
-          install_velox_deps_from_brew
-          install_double_conversion
+          install_velox_deps
 
           echo "NJOBS=`sysctl -n hw.ncpu`" >> $GITHUB_ENV
           brew unlink protobuf || echo "protobuf not installed"

diff --git a/scripts/setup-centos9.sh b/scripts/setup-centos9.sh
@@ -209,6 +209,11 @@ function install_cuda {
   dnf install -y cuda-nvcc-$(echo $1 | tr '.' '-') cuda-cudart-devel-$(echo $1 | tr '.' '-')
 }
 
+function install_proxygen {
+  wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen
+  cmake_install_dir proxygen -DBUILD_TESTS=OFF
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_dnf
   run_and_time install_conda
@@ -226,6 +231,7 @@ function install_velox_deps {
   run_and_time install_fbthrift
   run_and_time install_duckdb
   run_and_time install_arrow
+  run_and_time install_proxygen
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.

diff --git a/scripts/setup-macos.sh b/scripts/setup-macos.sh
@@ -39,10 +39,29 @@ export OS_CXXFLAGS=" -isystem $(brew --prefix)/include "
 NPROC=$(getconf _NPROCESSORS_ONLN)
 
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}
-MACOS_VELOX_DEPS="bison flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 snappy xz zstd"
+MACOS_VELOX_DEPS="bison boost flex gflags glog googletest icu4c libevent libsodium lz4 lzo openssl protobuf@21 snappy xz zstd"
 MACOS_BUILD_DEPS="ninja cmake"
 FB_OS_VERSION="v2024.05.20.00"
 FMT_VERSION="10.1.1"
+XSIMD_VERSION="10.0.0"
+ARROW_VERSION="15.0.0"
+DUCKDB_VERSION="v0.8.1"
+SIMDJSON_VERSION="3.9.3"
+PROTOBUF_VERSION="21.8"
+
+# Adapter related versions.
+ABSEIL_VERSION="20240116.2"
+GRPC_VERSION="v1.48.1"
+CRC32_VERSION="1.1.2"
+NLOHMAN_JSON_VERSION="v3.11.3"
+GOOGLE_CLOUD_CPP_VERSION="v2.22.0"
+HADOOP_VERSION="2.10.1"
+AZURE_SDK_VERSION="12.8.0"
+
+
+CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
+USE_CLANG="${USE_CLANG:-false}"
 
 function update_brew {
   DEFAULT_BREW_PATH=/usr/local/bin/brew
@@ -96,16 +115,66 @@ function install_velox_deps_from_brew {
   done
 }
 
+function install_xsimd {
+  wget_and_untar https://github.com/xtensor-stack/xsimd/archive/refs/tags/${XSIMD_VERSION}.tar.gz xsimd
+  cmake_install_dir xsimd
+}
+
 function install_fmt {
   wget_and_untar https://github.com/fmtlib/fmt/archive/${FMT_VERSION}.tar.gz fmt
   cmake_install_dir fmt -DFMT_TEST=OFF
 }
 
+function install_arrow {
+   set -e  # Exit on first error
+   echo "Downloading and extracting Apache Arrow version ${ARROW_VERSION}..."
+   wget_and_untar https://archive.apache.org/dist/arrow/arrow-${ARROW_VERSION}/apache-arrow-${ARROW_VERSION}.tar.gz arrow
+
+   echo "Starting the build process..."
+   (
+     cd arrow/cpp
+     cmake_install_dir \
+       -DARROW_PARQUET=OFF \
+       -DARROW_WITH_THRIFT=ON \
+       -DARROW_WITH_LZ4=ON \
+       -DARROW_WITH_SNAPPY=ON \
+       -DARROW_WITH_ZLIB=ON \
+       -DARROW_WITH_ZSTD=ON \
+       -DARROW_JEMALLOC=OFF \
+       -DARROW_SIMD_LEVEL=NONE \
+       -DARROW_RUNTIME_SIMD_LEVEL=NONE \
+       -DARROW_WITH_UTF8PROC=OFF \
+       -DARROW_TESTING=ON \
+       -Dcmake_install_dir_PREFIX=/usr/local \
+       -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
+       -DARROW_BUILD_STATIC=ON \
+       -DThrift_SOURCE=BUNDLED
+
+     echo "Installing Thrift..."
+     cd _build/thrift_ep-prefix/src/thrift_ep-build
+     ${SUDO} cmake --install ./ --prefix /usr/local/
+   )
+   echo "Apache Arrow installation complete."
+}
+
+function install_duckdb {
+  if $BUILD_DUCKDB ; then
+    echo 'Building DuckDB'
+    wget_and_untar https://github.com/duckdb/duckdb/archive/refs/tags/${DUCKDB_VERSION}.tar.gz duckdb
+    cmake_install_dir duckdb -DBUILD_UNITTESTS=OFF -DENABLE_SANITIZER=OFF -DENABLE_UBSAN=OFF -DBUILD_SHELL=OFF -DEXPORT_DLL_SYMBOLS=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
+  fi
+}
+
 function install_folly {
   wget_and_untar https://github.com/facebook/folly/archive/refs/tags/${FB_OS_VERSION}.tar.gz folly
   cmake_install_dir folly -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON
 }
 
+function install_protobuf {
+  wget_and_untar https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION}/protobuf-all-${PROTOBUF_VERSION}.tar.gz protobuf
+  cmake_install_dir protobuf -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=package
+}
+
 function install_fizz {
   wget_and_untar https://github.com/facebookincubator/fizz/archive/refs/tags/${FB_OS_VERSION}.tar.gz fizz
   cmake_install_dir fizz/fizz -DBUILD_TESTS=OFF
@@ -141,17 +210,79 @@ function install_re2 {
   cmake_install_dir re2 -DRE2_BUILD_TESTING=OFF
 }
 
+function install_proxygen {
+  wget_and_untar https://github.com/facebook/proxygen/archive/refs/tags/${FB_OS_VERSION}.tar.gz proxygen
+  cmake_install_dir proxygen -DBUILD_TESTS=OFF
+}
+
+function install_simdjson {
+  wget_and_untar https://github.com/simdjson/simdjson/archive/refs/tags/v${SIMDJSON_VERSION}.tar.gz simdjson
+  cmake_install_dir simdjson
+}
+
+function install_gcs {
+  # Install gcs dependencies
+  # https://github.com/googleapis/google-cloud-cpp/blob/main/doc/packaging.md#required-libraries
+
+  # abseil-cpp
+  github_checkout abseil/abseil-cpp ${ABSEIL_VERSION} --depth 1
+  cmake_install_dir \
+    -DABSL_BUILD_TESTING=OFF \
+    -DCMAKE_CXX_STANDARD=17 \
+    -DABSL_PROPAGATE_CXX_STD=ON \
+    -DABSL_ENABLE_INSTALL=ON
+
+  # protobuf
+  github_checkout protocolbuffers/protobuf v${PROTOBUF_VERSION} --depth 1
+  cmake_install_dir \
+    -Dprotobuf_BUILD_TESTS=OFF \
+    -Dprotobuf_ABSL_PROVIDER=package
+
+  # grpc
+  github_checkout grpc/grpc ${GRPC_VERSION} --depth 1
+  cmake_install_dir \
+    -DgRPC_BUILD_TESTS=OFF \
+    -DgRPC_ABSL_PROVIDER=package \
+    -DgRPC_ZLIB_PROVIDER=package \
+    -DgRPC_CARES_PROVIDER=package \
+    -DgRPC_RE2_PROVIDER=package \
+    -DgRPC_SSL_PROVIDER=package \
+    -DgRPC_PROTOBUF_PROVIDER=package \
+    -DgRPC_INSTALL=ON
+
+  # crc32
+  github_checkout google/crc32c ${CRC32_VERSION} --depth 1
+  cmake_install_dir \
+    -DCRC32C_BUILD_TESTS=OFF \
+    -DCRC32C_BUILD_BENCHMARKS=OFF \
+    -DCRC32C_USE_GLOG=OFF
+
+  # nlohmann json
+  github_checkout nlohmann/json ${NLOHMAN_JSON_VERSION} --depth 1
+  cmake_install_dir \
+    -DJSON_BuildTests=OFF
+
+  # google-cloud-cpp
+  github_checkout googleapis/google-cloud-cpp ${GOOGLE_CLOUD_CPP_VERSION} --depth 1
+  cmake_install_dir \
+    -DGOOGLE_CLOUD_CPP_ENABLE_EXAMPLES=OFF \
+    -DGOOGLE_CLOUD_CPP_ENABLE=storage
+}
+
 function install_velox_deps {
   run_and_time install_velox_deps_from_brew
-  run_and_time install_ranges_v3
   run_and_time install_double_conversion
-  run_and_time install_re2
   run_and_time install_fmt
   run_and_time install_folly
   run_and_time install_fizz
   run_and_time install_wangle
   run_and_time install_mvfst
   run_and_time install_fbthrift
+  run_and_time install_xsimd
+  run_and_time install_proxygen
+  run_and_time install_arrow
+  run_and_time install_duckdb
+  run_and_time install_simdjson
 }
 
 (return 2> /dev/null) && return # If script was sourced, don't run commands.