From f7a0092d21fa4539e6145141e41f6e4ba2d2e39a Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Thu, 17 Oct 2024 16:42:51 +0800 Subject: [PATCH] Initial --- .github/workflows/velox_weekly.yml | 9 +++++++-- dev/build_helper_functions.sh | 6 +++--- dev/ci-velox-buildshared-centos-8.sh | 2 +- dev/ci-velox-buildstatic-centos-7.sh | 2 +- dev/docker/Dockerfile.centos7-static-build | 4 ++-- dev/docker/Dockerfile.centos8-dynamic-build | 6 +++--- dev/docker/entrypoint.sh | 2 +- dev/package-vcpkg.sh | 4 ++-- dev/vcpkg/README.md | 6 +++--- ep/build-velox/src/setup-centos7.sh | 14 +++----------- ep/build-velox/src/setup-centos8.sh | 14 +++++++------- 11 files changed, 33 insertions(+), 36 deletions(-) diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 96de3bd570c8b..d0d1b27ebcd14 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -49,6 +49,7 @@ jobs: yum install -y epel-release sudo dnf if [ "${{ matrix.os }}" = "centos:7" ]; then yum install -y centos-release-scl + source /opt/rh/devtoolset-11/enable rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f sed -i \ -e 's/^mirrorlist/#mirrorlist/' \ @@ -59,8 +60,8 @@ jobs: dnf install -y --setopt=install_weak_deps=False gcc-toolset-12 source /opt/rh/gcc-toolset-12/enable || exit 1 else - dnf install -y --setopt=install_weak_deps=False gcc-toolset-9 - source /opt/rh/gcc-toolset-9/enable || exit 1 + dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 + source /opt/rh/gcc-toolset-11/enable || exit 1 fi yum install -y java-1.8.0-openjdk-devel patch wget git perl export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ @@ -87,6 +88,10 @@ jobs: # To avoid the prompt for region selection during installing tzdata. export DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y sudo maven wget git + if [ "${{ matrix.os }}" = "centos:7" ]; then + sudo apt-get install software-properties-common + sudo add-apt-repository ppa:ubuntu-toolchain-r/test + fi sudo apt-get install -y openjdk-8-jdk export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64 cd $GITHUB_WORKSPACE/ && ./dev/package.sh diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh index b69a36c29b1cd..97e3a09935206 100644 --- a/dev/build_helper_functions.sh +++ b/dev/build_helper_functions.sh @@ -165,7 +165,7 @@ function setup_linux { $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH - source /opt/rh/devtoolset-9/enable + source /opt/rh/devtoolset-11/enable set -u ;; *) @@ -179,7 +179,7 @@ function setup_linux { $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH - source /opt/rh/devtoolset-9/enable + source /opt/rh/devtoolset-11/enable set -u ;; 3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;; @@ -194,7 +194,7 @@ function setup_linux { $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh set +u export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH - source /opt/rh/devtoolset-9/enable + source /opt/rh/devtoolset-11/enable set -u ;; 3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;; diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh index af52d6554971d..ff9a62b798b5a 100755 --- a/dev/ci-velox-buildshared-centos-8.sh +++ b/dev/ci-velox-buildshared-centos-8.sh @@ -2,6 +2,6 @@ set -e -source /opt/rh/gcc-toolset-9/enable +source /opt/rh/gcc-toolset-11/enable ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_ep_cache=OFF --build_tests=ON \ --build_examples=ON --build_benchmarks=ON diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh index 3272de95d910d..6895891a98e98 100755 --- a/dev/ci-velox-buildstatic-centos-7.sh +++ b/dev/ci-velox-buildstatic-centos-7.sh @@ -2,7 +2,7 @@ set -e -source /opt/rh/devtoolset-9/enable +source /opt/rh/devtoolset-11/enable export NUM_THREADS=4 ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \ --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build index 5fb164189a2b7..a0da8bda5b472 100644 --- a/dev/docker/Dockerfile.centos7-static-build +++ b/dev/docker/Dockerfile.centos7-static-build @@ -11,7 +11,7 @@ RUN sed -i \ -e 's/mirror\.centos\.org/vault.centos.org/' \ /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo -RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel +RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel ccache RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten @@ -26,7 +26,7 @@ RUN mkdir -p /var/cache/vcpkg ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite # Build arrow, then install the native libs to system paths and jar package to .m2/ directory. -RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && \ +RUN cd /opt/gluten && source /opt/rh/devtoolset-11/enable && \ bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \ --enable_hdfs=ON --enable_abfs=ON build_arrow && \ rm -rf /opt/gluten diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build index 860a01a54ccea..1c85ccaf83377 100644 --- a/dev/docker/Dockerfile.centos8-dynamic-build +++ b/dev/docker/Dockerfile.centos8-dynamic-build @@ -3,8 +3,8 @@ FROM centos:8 RUN sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true RUN sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true -RUN yum update -y && yum install -y epel-release sudo dnf -RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-9 +RUN yum update -y && yum install -y epel-release sudo dnf ccache +RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-11 RUN echo "check_certificate = off" >> ~/.wgetrc RUN yum install -y java-1.8.0-openjdk-devel patch wget git perl @@ -17,4 +17,4 @@ ENV PATH=${PATH}:/usr/lib/maven/bin RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten -RUN cd /opt/gluten && source /opt/rh/gcc-toolset-9/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten +RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten diff --git a/dev/docker/entrypoint.sh b/dev/docker/entrypoint.sh index 99703a388d232..be452cac12732 100755 --- a/dev/docker/entrypoint.sh +++ b/dev/docker/entrypoint.sh @@ -1,4 +1,4 @@ #! /bin/sh -. /opt/rh/devtoolset-9/enable +. /opt/rh/devtoolset-11/enable exec "$@" \ No newline at end of file diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh index 7d5de3e948b16..acc69560c0db1 100755 --- a/dev/package-vcpkg.sh +++ b/dev/package-vcpkg.sh @@ -11,10 +11,10 @@ ARCH=`uname -m` cd "$GLUTEN_DIR" if [ "$LINUX_OS" == "centos" ]; then if [ "$VERSION" == "8" ]; then - source /opt/rh/gcc-toolset-9/enable + source /opt/rh/gcc-toolset-11/enable elif [ "$VERSION" == "7" ]; then export MANPATH="" - source /opt/rh/devtoolset-9/enable + source /opt/rh/devtoolset-11/enable fi fi diff --git a/dev/vcpkg/README.md b/dev/vcpkg/README.md index c5f1dc4a507be..229e11cc72047 100644 --- a/dev/vcpkg/README.md +++ b/dev/vcpkg/README.md @@ -15,14 +15,14 @@ Please install build depends on your system to compile all libraries: sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh ``` -For CentOS user, gcc 9 needs to be enabled manually before next step: +For CentOS user, gcc 11 needs to be enabled manually before next step: ``` sh # CentOS 8 -source /opt/rh/gcc-toolset-9/enable +source /opt/rh/gcc-toolset-11/enable # CentOS 7 -source /opt/rh/devtoolset-9/enable +source /opt/rh/devtoolset-11/enable ``` For unsupported linux distro, you can install the following packages from package manager. diff --git a/ep/build-velox/src/setup-centos7.sh b/ep/build-velox/src/setup-centos7.sh index 708c692ea5827..753c31c260d95 100755 --- a/ep/build-velox/src/setup-centos7.sh +++ b/ep/build-velox/src/setup-centos7.sh @@ -157,7 +157,7 @@ function install_lzo { function install_boost { # Remove old version. - sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-9/root/usr/lib64/dyninst/libboost_* + sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-11/root/usr/lib64/dyninst/libboost_* sudo rm -rf /tmp/velox-deps/boost/ /usr/local/include/boost/ /usr/local/lib/cmake/Boost-1.72.0/ cd "${DEPENDENCY_DIR}" wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost @@ -239,8 +239,6 @@ $SUDO dnf makecache # dnf install dependency libraries dnf_install epel-release dnf-plugins-core # For ccache, ninja -# PowerTools only works on CentOS8 -# dnf config-manager --set-enabled powertools dnf_install ccache wget which libevent-devel \ yasm \ openssl-devel libzstd-devel lz4-devel double-conversion-devel \ @@ -254,16 +252,10 @@ dnf_install autoconf automake libtool bison python3 python3-devel # Required for build flex dnf_install gettext-devel texinfo help2man -# dnf_install conda - -# Activate gcc9; enable errors on unset variables afterwards. -# GCC9 install via yum and devtoolset -# dnf install gcc-toolset-9 only works on CentOS8 - $SUDO yum makecache yum_install centos-release-scl -yum_install devtoolset-9 -source /opt/rh/devtoolset-9/enable || exit 1 +yum_install devtoolset-11 +source /opt/rh/devtoolset-11/enable || exit 1 gcc --version set -u diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh index 7ad6560653885..c9d916c641c0b 100755 --- a/ep/build-velox/src/setup-centos8.sh +++ b/ep/build-velox/src/setup-centos8.sh @@ -38,8 +38,8 @@ export CXXFLAGS=$CFLAGS # Used by boost. export CPPFLAGS=$CFLAGS # Used by LZO. CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}" BUILD_DUCKDB="${BUILD_DUCKDB:-true}" -export CC=/opt/rh/gcc-toolset-9/root/bin/gcc -export CXX=/opt/rh/gcc-toolset-9/root/bin/g++ +export CC=/opt/rh/gcc-toolset-11/root/bin/gcc +export CXX=/opt/rh/gcc-toolset-11/root/bin/g++ DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download} FB_OS_VERSION="v2024.05.20.00" @@ -56,7 +56,7 @@ function install_build_prerequisites { dnf_install epel-release dnf-plugins-core # For ccache, ninja dnf config-manager --set-enabled powertools dnf update -y - dnf_install ninja-build curl ccache gcc-toolset-9 git wget which + dnf_install ninja-build curl ccache gcc-toolset-11 git wget which dnf_install yasm dnf_install autoconf automake python39 python39-devel python39-pip libtool pip3.9 install cmake==3.28.3 @@ -198,8 +198,8 @@ function install_velox_deps { ( if [[ $# -ne 0 ]]; then - # Activate gcc9; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-9/enable || exit 1 + # Activate gcc11; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-11/enable || exit 1 set -u for cmd in "$@"; do run_and_time "${cmd}" @@ -212,8 +212,8 @@ function install_velox_deps { else echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set" fi - # Activate gcc9; enable errors on unset variables afterwards. - source /opt/rh/gcc-toolset-9/enable || exit 1 + # Activate gcc11; enable errors on unset variables afterwards. + source /opt/rh/gcc-toolset-11/enable || exit 1 set -u install_velox_deps echo "All dependencies for Velox installed!"