Initial

apache · Oct 17, 2024 · f7a0092 · f7a0092
1 parent a7908ff
commit f7a0092
Show file tree

Hide file tree

Showing 11 changed files with 33 additions and 36 deletions.
diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml
@@ -49,6 +49,7 @@ jobs:
           yum install -y epel-release sudo dnf
           if [ "${{ matrix.os }}" = "centos:7" ]; then
             yum install -y centos-release-scl
+            source /opt/rh/devtoolset-11/enable
             rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
             sed -i \
             -e 's/^mirrorlist/#mirrorlist/' \
@@ -59,8 +60,8 @@ jobs:
             dnf install -y --setopt=install_weak_deps=False gcc-toolset-12
             source /opt/rh/gcc-toolset-12/enable || exit 1
           else
-            dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
-            source /opt/rh/gcc-toolset-9/enable || exit 1
+            dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
+            source /opt/rh/gcc-toolset-11/enable || exit 1
           fi
           yum install -y java-1.8.0-openjdk-devel patch wget git perl
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
@@ -87,6 +88,10 @@ jobs:
           # To avoid the prompt for region selection during installing tzdata.
           export DEBIAN_FRONTEND=noninteractive
           apt-get update && apt-get install -y sudo maven wget git
+          if [ "${{ matrix.os }}" = "centos:7" ]; then
+            sudo apt-get install software-properties-common
+            sudo add-apt-repository ppa:ubuntu-toolchain-r/test
+          fi
           sudo apt-get install -y openjdk-8-jdk
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh
diff --git a/dev/build_helper_functions.sh b/dev/build_helper_functions.sh
@@ -165,7 +165,7 @@ function setup_linux {
       $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-      source /opt/rh/devtoolset-9/enable
+      source /opt/rh/devtoolset-11/enable
       set -u
       ;;
     *)
@@ -179,7 +179,7 @@ function setup_linux {
       $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
       set +u
       export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-      source /opt/rh/devtoolset-9/enable
+      source /opt/rh/devtoolset-11/enable
       set -u
       ;;
     3) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;
@@ -194,7 +194,7 @@ function setup_linux {
         $GLUTEN_VELOX_SCRIPT_HOME/setup-centos7.sh
         set +u
         export PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig:$PKG_CONFIG_PATH
-        source /opt/rh/devtoolset-9/enable
+        source /opt/rh/devtoolset-11/enable
         set -u
         ;;
     3.2) $GLUTEN_VELOX_SCRIPT_HOME/setup-centos8.sh ;;

diff --git a/dev/ci-velox-buildshared-centos-8.sh b/dev/ci-velox-buildshared-centos-8.sh
@@ -2,6 +2,6 @@
 
 set -e
 
-source /opt/rh/gcc-toolset-9/enable
+source /opt/rh/gcc-toolset-11/enable
 ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --build_arrow=OFF --enable_ep_cache=OFF --build_tests=ON \
     --build_examples=ON --build_benchmarks=ON
diff --git a/dev/ci-velox-buildstatic-centos-7.sh b/dev/ci-velox-buildstatic-centos-7.sh
@@ -2,7 +2,7 @@
 
 set -e
 
-source /opt/rh/devtoolset-9/enable
+source /opt/rh/devtoolset-11/enable
 export NUM_THREADS=4
 ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_arrow=OFF --build_tests=OFF --build_benchmarks=OFF \
                            --build_examples=OFF --enable_s3=ON --enable_gcs=ON --enable_hdfs=ON --enable_abfs=ON
diff --git a/dev/docker/Dockerfile.centos7-static-build b/dev/docker/Dockerfile.centos7-static-build
@@ -11,7 +11,7 @@ RUN sed -i \
        -e 's/mirror\.centos\.org/vault.centos.org/' \
        /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
 
-RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel
+RUN yum install -y git patch wget sudo java-1.8.0-openjdk-devel ccache
 
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
@@ -26,7 +26,7 @@ RUN mkdir -p /var/cache/vcpkg
 ENV VCPKG_BINARY_SOURCES=clear;files,/var/cache/vcpkg,readwrite
 
 # Build arrow, then install the native libs to system paths and jar package to .m2/ directory.
-RUN cd /opt/gluten && source /opt/rh/devtoolset-9/enable && \
+RUN cd /opt/gluten && source /opt/rh/devtoolset-11/enable && \
     bash ./dev/builddeps-veloxbe.sh --enable_vcpkg=ON --build_tests=ON --enable_s3=ON --enable_gcs=ON \
                                     --enable_hdfs=ON --enable_abfs=ON  build_arrow && \
     rm -rf /opt/gluten
diff --git a/dev/docker/Dockerfile.centos8-dynamic-build b/dev/docker/Dockerfile.centos8-dynamic-build
@@ -3,8 +3,8 @@ FROM centos:8
 RUN sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
 RUN sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
 
-RUN yum update -y && yum install -y epel-release sudo dnf
-RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
+RUN yum update -y && yum install -y epel-release sudo dnf ccache
+RUN dnf install -y --setopt=install_weak_deps=False gcc-toolset-11
 RUN echo "check_certificate = off" >> ~/.wgetrc
 
 RUN yum install -y java-1.8.0-openjdk-devel patch wget git perl
@@ -17,4 +17,4 @@ ENV PATH=${PATH}:/usr/lib/maven/bin
 
 RUN git clone --depth=1 https://github.com/apache/incubator-gluten /opt/gluten
 
-RUN cd /opt/gluten && source /opt/rh/gcc-toolset-9/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten
+RUN cd /opt/gluten && source /opt/rh/gcc-toolset-11/enable && ./dev/builddeps-veloxbe.sh --run_setup_script=ON build_arrow && rm -rf /opt/gluten
diff --git a/dev/docker/entrypoint.sh b/dev/docker/entrypoint.sh
@@ -1,4 +1,4 @@
 #! /bin/sh
 
-. /opt/rh/devtoolset-9/enable
+. /opt/rh/devtoolset-11/enable
 exec "$@"
diff --git a/dev/package-vcpkg.sh b/dev/package-vcpkg.sh
@@ -11,10 +11,10 @@ ARCH=`uname -m`
 cd "$GLUTEN_DIR"
 if [ "$LINUX_OS" == "centos" ]; then
   if [ "$VERSION" == "8" ]; then
-    source /opt/rh/gcc-toolset-9/enable
+    source /opt/rh/gcc-toolset-11/enable
   elif [ "$VERSION" == "7" ]; then
     export MANPATH=""
-    source /opt/rh/devtoolset-9/enable
+    source /opt/rh/devtoolset-11/enable
   fi
 fi
 

diff --git a/dev/vcpkg/README.md b/dev/vcpkg/README.md
@@ -15,14 +15,14 @@ Please install build depends on your system to compile all libraries:
 sudo $GLUTEN_REPO/dev/vcpkg/setup-build-depends.sh
 ```
 
-For CentOS user, gcc 9 needs to be enabled manually before next step:
+For CentOS user, gcc 11 needs to be enabled manually before next step:
 
 ``` sh
 # CentOS 8
-source /opt/rh/gcc-toolset-9/enable
+source /opt/rh/gcc-toolset-11/enable
 
 # CentOS 7
-source /opt/rh/devtoolset-9/enable
+source /opt/rh/devtoolset-11/enable
 ```
 
 For unsupported linux distro, you can install the following packages from package manager.

diff --git a/ep/build-velox/src/setup-centos7.sh b/ep/build-velox/src/setup-centos7.sh
@@ -157,7 +157,7 @@ function install_lzo {
 
 function install_boost {
   # Remove old version.
-  sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-9/root/usr/lib64/dyninst/libboost_*
+  sudo rm -f /usr/local/lib/libboost_* /usr/lib64/libboost_* /opt/rh/devtoolset-11/root/usr/lib64/dyninst/libboost_*
   sudo rm -rf /tmp/velox-deps/boost/ /usr/local/include/boost/ /usr/local/lib/cmake/Boost-1.72.0/
   cd "${DEPENDENCY_DIR}"
   wget_and_untar https://github.com/boostorg/boost/releases/download/boost-1.84.0/boost-1.84.0.tar.gz boost
@@ -239,8 +239,6 @@ $SUDO dnf makecache
 
 # dnf install dependency libraries
 dnf_install epel-release dnf-plugins-core # For ccache, ninja
-# PowerTools only works on CentOS8
-# dnf config-manager --set-enabled powertools
 dnf_install ccache wget which libevent-devel \
   yasm \
   openssl-devel libzstd-devel lz4-devel double-conversion-devel \
@@ -254,16 +252,10 @@ dnf_install autoconf automake libtool bison python3 python3-devel
 # Required for build flex
 dnf_install gettext-devel texinfo help2man
 
-# dnf_install conda
-
-# Activate gcc9; enable errors on unset variables afterwards.
-# GCC9 install via yum and devtoolset
-# dnf install gcc-toolset-9 only works on CentOS8
-
 $SUDO yum makecache
 yum_install centos-release-scl
-yum_install devtoolset-9
-source /opt/rh/devtoolset-9/enable || exit 1
+yum_install devtoolset-11
+source /opt/rh/devtoolset-11/enable || exit 1
 gcc --version
 set -u
 

diff --git a/ep/build-velox/src/setup-centos8.sh b/ep/build-velox/src/setup-centos8.sh
@@ -38,8 +38,8 @@ export CXXFLAGS=$CFLAGS  # Used by boost.
 export CPPFLAGS=$CFLAGS  # Used by LZO.
 CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
 BUILD_DUCKDB="${BUILD_DUCKDB:-true}"
-export CC=/opt/rh/gcc-toolset-9/root/bin/gcc
-export CXX=/opt/rh/gcc-toolset-9/root/bin/g++
+export CC=/opt/rh/gcc-toolset-11/root/bin/gcc
+export CXX=/opt/rh/gcc-toolset-11/root/bin/g++
 DEPENDENCY_DIR=${DEPENDENCY_DIR:-$(pwd)/deps-download}
 
 FB_OS_VERSION="v2024.05.20.00"
@@ -56,7 +56,7 @@ function install_build_prerequisites {
   dnf_install epel-release dnf-plugins-core # For ccache, ninja
   dnf config-manager --set-enabled powertools
   dnf update -y
-  dnf_install ninja-build curl ccache gcc-toolset-9 git wget which
+  dnf_install ninja-build curl ccache gcc-toolset-11 git wget which
   dnf_install yasm
   dnf_install autoconf automake python39 python39-devel python39-pip libtool
   pip3.9 install cmake==3.28.3
@@ -198,8 +198,8 @@ function install_velox_deps {
 
 (
   if [[ $# -ne 0 ]]; then
-    # Activate gcc9; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-9/enable || exit 1
+    # Activate gcc11; enable errors on unset variables afterwards.
+    source /opt/rh/gcc-toolset-11/enable || exit 1
     set -u
     for cmd in "$@"; do
       run_and_time "${cmd}"
@@ -212,8 +212,8 @@ function install_velox_deps {
     else
       echo "Skipping installation of build dependencies since INSTALL_PREREQUISITES is not set"
     fi
-    # Activate gcc9; enable errors on unset variables afterwards.
-    source /opt/rh/gcc-toolset-9/enable || exit 1
+    # Activate gcc11; enable errors on unset variables afterwards.
+    source /opt/rh/gcc-toolset-11/enable || exit 1
     set -u
     install_velox_deps
     echo "All dependencies for Velox installed!"