diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index fc21673eb45b..934c36ebb7cb 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -269,13 +269,13 @@ jobs: run: | docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' + ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF && \ + ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF' - name: Build Gluten CPP library run: | docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF' - name: Build for Spark 3.2.2 run: | docker exec ubuntu2204-test-$GITHUB_RUN_ID bash -c ' @@ -348,14 +348,14 @@ jobs: source /env.sh && \ sudo yum -y install patch && \ cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' + ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF && \ + ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF' - name: Build Gluten CPP library run: | docker exec centos8-test-$GITHUB_RUN_ID bash -c ' source /env.sh && \ cd /opt/gluten/cpp && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF' - name: Build for Spark 3.2.2 run: | docker exec centos8-test-$GITHUB_RUN_ID bash -c ' @@ -399,14 +399,14 @@ jobs: docker exec centos7-test-$GITHUB_RUN_ID bash -c ' yum -y install epel-release centos-release-scl patch sudo && \ cd /opt/gluten/ep/build-velox/src && \ - ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON && \ - ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' + ./get_velox.sh --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF && \ + ./build_velox.sh --velox_home=/opt/velox --enable_ep_cache=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF --enable_hdfs=ON' - name: Build Gluten CPP library run: | docker exec centos7-test-$GITHUB_RUN_ID bash -c ' cd /opt/gluten/cpp && \ source /opt/rh/devtoolset-9/enable && \ - ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON' + ./compile.sh --build_velox_backend=ON --velox_home=/opt/velox --enable_hdfs=ON --enable_s3=ON --enable_gcs=ON --enable_abfs=OFF' - name: Build for Spark 3.2.2 run: | docker exec centos7-test-$GITHUB_RUN_ID bash -c ' @@ -488,7 +488,7 @@ jobs: cd /opt/gluten && \ sudo -E ./dev/vcpkg/setup-build-depends.sh && \ source ./dev/vcpkg/env.sh && \ - ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON' + ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=OFF' - name: Build for Spark 3.2.2 run: | docker exec static-build-test-$GITHUB_RUN_ID bash -c ' diff --git a/dev/vcpkg/vcpkg.json b/dev/vcpkg/vcpkg.json index 91cb430fb879..e7491cf9d550 100644 --- a/dev/vcpkg/vcpkg.json +++ b/dev/vcpkg/vcpkg.json @@ -2,7 +2,7 @@ "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg.schema.json", "builtin-baseline": "a7b6122f6b6504d16d96117336a0562693579933", "dependencies": ["jemalloc"], - "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs"], + "default-features": ["velox", "velox-s3", "velox-gcs", "velox-hdfs", "velox-abfs"], "features": { "velox": { "description": "Velox backend", @@ -83,6 +83,12 @@ "libhdfs3" ] }, + "velox-abfs": { + "description": "Velox ABFS Support", + "dependencies": [ + "azure-storage-blobs-cpp" + ] + }, "hbm": { "description": "HBM", "dependencies": [ diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index e430dd7c1bb9..f1209fbba062 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -28,6 +28,8 @@ BUILD_PROTOBUF=ON ENABLE_S3=OFF #Set on run gluten on GCS ENABLE_GCS=OFF +#Set on run gluten on ABFS +ENABLE_ABFS=OFF OS=`uname -s` @@ -61,6 +63,10 @@ for arg in "$@"; do ENABLE_GCS=("${arg#*=}") shift # Remove argument name from processing ;; + --enable_abfs=*) + ENABLE_ABFS=("${arg#*=}") + shift # Remove argument name from processing + ;; *) OTHER_ARGUMENTS+=("$1") shift # Remove generic argument from processing @@ -99,6 +105,9 @@ function process_setup_ubuntu { if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-ubuntu.sh fi + if [ $ENABLE_ABFS == "ON" ]; then + sed -i '/^ run_and_time install_fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-ubuntu.sh + fi sed -i 's/run_and_time install_conda/#run_and_time install_conda/' scripts/setup-ubuntu.sh } @@ -129,6 +138,9 @@ function process_setup_centos8 { if [ $ENABLE_GCS == "ON" ]; then sed -i '/^cmake_install_deps fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos8.sh fi + if [ $ENABLE_ABFS == "ON" ]; then + sed -i '/^cmake_install_deps fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos8.sh + fi } function process_setup_centos7 { @@ -156,6 +168,9 @@ function process_setup_centos7 { if [ $ENABLE_GCS == "ON" ]; then sed -i '/^ run_and_time install_fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh gcs' scripts/setup-centos7.sh fi + if [ $ENABLE_ABFS == "ON" ]; then + sed -i '/^ run_and_time install_fmt/a \ \ '${VELOX_HOME}/scripts'/setup-adapters.sh abfs' scripts/setup-centos7.sh + fi } function process_setup_alinux3 {