From 09f92161b9ecd0127b4bb21c550d4e6dee8f1913 Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Tue, 12 Dec 2023 17:19:18 +0800 Subject: [PATCH] [VL] Fix git version conflict issue in build script (#4007) --- .github/workflows/velox_be.yml | 2 +- dev/builddeps-veloxbe.sh | 9 ++++++-- ep/build-velox/src/build_velox.sh | 12 +++++------ ep/build-velox/src/get_velox.sh | 31 +++++++++++++++++++++------ ep/build-velox/src/modify_velox.patch | 13 ----------- 5 files changed, 38 insertions(+), 29 deletions(-) diff --git a/.github/workflows/velox_be.yml b/.github/workflows/velox_be.yml index 7d4ac141133f..0fac2b87a249 100644 --- a/.github/workflows/velox_be.yml +++ b/.github/workflows/velox_be.yml @@ -488,7 +488,7 @@ jobs: cd /opt/gluten && \ sudo -E ./dev/vcpkg/setup-build-depends.sh && \ source ./dev/vcpkg/env.sh && \ - ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=OFF' + ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --build_benchmarks=ON --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=OFF' - name: Build for Spark 3.2.2 run: | docker exec static-build-test-$GITHUB_RUN_ID bash -c ' diff --git a/dev/builddeps-veloxbe.sh b/dev/builddeps-veloxbe.sh index 59260f49ad64..dc3274119038 100755 --- a/dev/builddeps-veloxbe.sh +++ b/dev/builddeps-veloxbe.sh @@ -24,6 +24,7 @@ ENABLE_ABFS=OFF ENABLE_EP_CACHE=OFF ARROW_ENABLE_CUSTOM_CODEC=OFF ENABLE_VCPKG=OFF +RUN_SETUP_SCRIPT=ON for arg in "$@" do @@ -89,6 +90,10 @@ do --enable_vcpkg=*) ENABLE_VCPKG=("${arg#*=}") shift # Remove argument name from processing + ;; + --run_setup_script=*) + RUN_SETUP_SCRIPT=("${arg#*=}") + shift # Remove argument name from processing ;; *) OTHER_ARGUMENTS+=("$1") @@ -106,8 +111,8 @@ fi ##install velox cd $GLUTEN_DIR/ep/build-velox/src ./get_velox.sh --enable_hdfs=$ENABLE_HDFS --build_protobuf=$BUILD_PROTOBUF --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --enable_abfs=$ENABLE_ABFS -./build_velox.sh --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS --enable_abfs=$ENABLE_ABFS \ - --enable_ep_cache=$ENABLE_EP_CACHE --build_tests=$BUILD_TESTS --build_benchmarks=$BUILD_BENCHMARKS +./build_velox.sh --run_setup_script=$RUN_SETUP_SCRIPT --enable_s3=$ENABLE_S3 --enable_gcs=$ENABLE_GCS --build_type=$BUILD_TYPE --enable_hdfs=$ENABLE_HDFS \ + --enable_abfs=$ENABLE_ABFS --enable_ep_cache=$ENABLE_EP_CACHE --build_tests=$BUILD_TESTS --build_benchmarks=$BUILD_BENCHMARKS ## compile gluten cpp cd $GLUTEN_DIR/cpp diff --git a/ep/build-velox/src/build_velox.sh b/ep/build-velox/src/build_velox.sh index 88c0d4456a7e..9a9c4a151f6b 100755 --- a/ep/build-velox/src/build_velox.sh +++ b/ep/build-velox/src/build_velox.sh @@ -90,7 +90,7 @@ function compile { elif [ $OS == 'Darwin' ]; then setup_macos else - echo "Unsupport kernel: $OS" + echo "Unsupported kernel: $OS" exit 1 fi fi @@ -126,7 +126,7 @@ function compile { elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' ]]; then CPU_TARGET=$ARCH make $COMPILE_TYPE EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}" else - echo "Unsupport arch: $ARCH" + echo "Unsupported arch: $ARCH" exit 1 fi @@ -208,7 +208,7 @@ function setup_linux { set -u ;; *) - echo "Unsupport centos version: $LINUX_VERSION_ID" + echo "Unsupported centos version: $LINUX_VERSION_ID" exit 1 ;; esac @@ -223,7 +223,7 @@ function setup_linux { ;; 3) scripts/setup-centos8.sh ;; *) - echo "Unsupport alinux version: $LINUX_VERSION_ID" + echo "Unsupported alinux version: $LINUX_VERSION_ID" exit 1 ;; esac @@ -231,12 +231,12 @@ function setup_linux { case "$LINUX_VERSION_ID" in 3.2) scripts/setup-centos8.sh ;; *) - echo "Unsupport tencentos version: $LINUX_VERSION_ID" + echo "Unsupported tencentos version: $LINUX_VERSION_ID" exit 1 ;; esac else - echo "Unsupport linux distribution: $LINUX_DISTRIBUTION" + echo "Unsupported linux distribution: $LINUX_DISTRIBUTION" exit 1 fi } diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 046a66ce591e..ee74baf7b067 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -75,8 +75,16 @@ for arg in "$@"; do done function process_setup_ubuntu { - # make this function Reentrantly + if [ -z "$(which git)" ]; then + sudo --preserve-env apt install -y git + fi + # make this function Reentrant git checkout scripts/setup-ubuntu.sh + + # No need to re-install git. + sed -i '/git \\/d' scripts/setup-ubuntu.sh + # Do not install libunwind which can cause interruption when catching native exception. + sed -i 's/sudo --preserve-env apt install -y libunwind-dev && //' scripts/setup-ubuntu.sh sed -i '/libprotobuf-dev/d' scripts/setup-ubuntu.sh sed -i '/protobuf-compiler/d' scripts/setup-ubuntu.sh sed -i '/ccache/a\ *thrift* \\' scripts/setup-ubuntu.sh @@ -113,8 +121,15 @@ function process_setup_ubuntu { } function process_setup_centos8 { - # make this function Reentrantly + # Allows other version of git already installed. + if [ -z "$(which git)" ]; then + dnf install -y -q --setopt=install_weak_deps=False git + fi + # make this function Reentrant git checkout scripts/setup-centos8.sh + + # # No need to re-install git. + sed -i 's/dnf_install ninja-build ccache gcc-toolset-9 git/dnf_install ninja-build ccache gcc-toolset-9/' scripts/setup-centos8.sh sed -i '/^function dnf_install/i\DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)}' scripts/setup-centos8.sh sed -i '/^dnf_install autoconf/a\dnf_install libxml2-devel libgsasl-devel libuuid-devel' scripts/setup-centos8.sh sed -i '/^function cmake_install_deps.*/i FB_OS_VERSION=v2022.11.14.00\n function install_folly {\n github_checkout facebook/folly "${FB_OS_VERSION}"\n cmake_install -DBUILD_TESTS=OFF -DFOLLY_HAVE_INT128_T=ON\n}\n' scripts/setup-centos8.sh @@ -144,19 +159,21 @@ function process_setup_centos8 { } function process_setup_centos7 { - # make this function Reentrantly + # Allows other version of git already installed. + if [ -z "$(which git)" ]; then + dnf install -y -q --setopt=install_weak_deps=False git + fi + # make this function Reentrant git checkout scripts/setup-centos7.sh + # No need to re-install git. + sed -i 's/dnf_install ccache git/dnf_install ccache/' scripts/setup-centos7.sh # cmake 3 and ninja should be installed sed -i '/^run_and_time install_cmake/d' scripts/setup-centos7.sh sed -i '/^run_and_time install_ninja/d' scripts/setup-centos7.sh # install gtest sed -i '/^ run_and_time install_fmt/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh - # Don't force to install git. - sed -i 's/dnf_install ccache git/dnf_install ccache/' scripts/setup-centos7.sh - # Install git with --skip-broken to accept other version of git already installed. - sed -i '/^$SUDO dnf remove -y gflags/a \\dnf install -y -q --setopt=install_weak_deps=False --skip-broken git' scripts/setup-centos7.sh if [ $ENABLE_HDFS = "ON" ]; then sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh diff --git a/ep/build-velox/src/modify_velox.patch b/ep/build-velox/src/modify_velox.patch index 3aea6f332245..66aa01f82f25 100644 --- a/ep/build-velox/src/modify_velox.patch +++ b/ep/build-velox/src/modify_velox.patch @@ -79,19 +79,6 @@ index e2ff67fa5..e03b246e3 100644 if(NOT TARGET gflags::gflags) # This is a bit convoluted, but we want to be able to use gflags::gflags as a # target even when velox is built as a subproject which uses -diff --git a/scripts/setup-ubuntu.sh b/scripts/setup-ubuntu.sh -index 42054e593..4c793b785 100755 ---- a/scripts/setup-ubuntu.sh -+++ b/scripts/setup-ubuntu.sh -@@ -31,7 +31,7 @@ export CMAKE_BUILD_TYPE=Release - # Install all velox and folly dependencies. - # The is an issue on 22.04 where a version conflict prevents glog install, - # installing libunwind first fixes this. --sudo --preserve-env apt update && sudo --preserve-env apt install -y libunwind-dev && \ -+sudo --preserve-env apt update && \ - sudo --preserve-env apt install -y \ - g++ \ - cmake \ diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt index c7c4d8557..2715e625c 100644 --- a/third_party/CMakeLists.txt