[CI][VL] Re-enable a build job running on clean dockers weekly (apach…

…e#6424)

This is to make sure no build issue in Gluten + Velox. Other CI jobs use dockers with dependencies pre-installed, which doesn't uncover some build issues sometimes.

This job is scheduled to be triggered per week. Also fixes some issues found by this job.

.github/workflows/velox_nightly.yml.disabled → .github/workflows/velox_weekly.yml

@@ -13,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Velox backend nightly job
+name: Velox backend weekly job
 
 on:
   pull_request:
     paths:
-      - '.github/workflows/velox_nightly.yml'
+      - '.github/workflows/velox_weekly.yml'
   schedule:
-    - cron: '0 20 * * *'
+    - cron: '0 20 * * 0'
+
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -37,25 +40,28 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Update mirror list
-        if: matrix.os == 'centos:8'
         run: |
           sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
           sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
       - name: build
         run: |
           yum update -y
           yum install -y epel-release sudo dnf
-          if [ "${{ matrix.os }}" = "centos:8" ]; then
+          if [ "${{ matrix.os }}" = "centos:7" ]; then
+            yum install -y centos-release-scl
+            rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
+            sed -i \
+            -e 's/^mirrorlist/#mirrorlist/' \
+            -e 's/^#baseurl/baseurl/' \
+            -e 's/mirror\.centos\.org/vault.centos.org/' \
+            /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
+          else
             dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
             source /opt/rh/gcc-toolset-9/enable || exit 1
-          else
-            yum install -y centos-release-scl
-            yum install -y devtoolset-9
-            source /opt/rh/devtoolset-9/enable || exit 1
           fi
-          yum install -y java-1.8.0-openjdk-devel patch wget git && \
+          yum install -y java-1.8.0-openjdk-devel patch wget git
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
-          export PATH=$JAVA_HOME/bin:$PATH && \
+          export PATH=$JAVA_HOME/bin:$PATH
           wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
           tar -xvf apache-maven-3.8.8-bin.tar.gz && \
           mv apache-maven-3.8.8 /usr/lib/maven && \
@@ -76,7 +82,8 @@ jobs:
       - name: build
         run: |
           # To avoid the prompt for region selection during installing tzdata.
-          export DEBIAN_FRONTEND="noninteractive"
-          apt-get update && apt-get install -y sudo openjdk-8-jdk maven wget git
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update && apt-get install -y sudo maven wget git
+          sudo apt-get install -y openjdk-8-jdk
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh

dev/builddeps-veloxbe.sh

@@ -240,18 +240,17 @@ fi
 OS=`uname -s`
 source $GLUTEN_DIR/dev/build_helper_functions.sh
 if [ -z "${GLUTEN_VCPKG_ENABLED:-}" ] && [ $RUN_SETUP_SCRIPT == "ON" ]; then
-  (
-    echo "Start to install dependencies"
-    cd $VELOX_HOME
-    if [ $OS == 'Linux' ]; then
-      setup_linux
-    elif [ $OS == 'Darwin' ]; then
-      setup_macos
-    else
-      echo "Unsupported kernel: $OS"
-      exit 1
-    fi
-  )
+  echo "Start to install dependencies"
+  pushd $VELOX_HOME
+  if [ $OS == 'Linux' ]; then
+    setup_linux
+  elif [ $OS == 'Darwin' ]; then
+    setup_macos
+  else
+    echo "Unsupported kernel: $OS"
+    exit 1
+  fi
+  popd
 fi
 
 commands_to_run=${OTHER_ARGUMENTS:-}

ep/build-velox/src/get_velox.sh

@@ -94,7 +94,7 @@ function process_setup_ubuntu {
   sed -i '/^  run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-ubuntu.sh
   if [ $ENABLE_HDFS == "ON" ]; then
     sed -i '/^function install_folly.*/i function install_libhdfs3 {\n  github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-ubuntu.sh
-    sed -i '/^  run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh
+    sed -i '/^  run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-ubuntu.sh
     sed -i '/ccache /a\  yasm \\' scripts/setup-ubuntu.sh
   fi
   sed -i "s/apt install -y/sudo apt install -y/" ${VELOX_HOME}/scripts/setup-adapters.sh
@@ -166,9 +166,10 @@ function process_setup_centos7 {
   # install gtest
   sed -i '/^  run_and_time install_folly/a \ \ run_and_time install_gtest' scripts/setup-centos7.sh
   sed -i '/^  run_and_time install_folly/a \ \ run_and_time install_protobuf' scripts/setup-centos7.sh
+  sed -i 's/https:\/\/cmake.org\/files\/v3.25\/cmake-3.25.1.tar.gz/https:\/\/cmake.org\/files\/v3.28\/cmake-3.28.3.tar.gz/' scripts/setup-centos7.sh
   if [ $ENABLE_HDFS = "ON" ]; then
     sed -i '/^function install_protobuf.*/i function install_libhdfs3 {\n cd "\${DEPENDENCY_DIR}"\n github_checkout oap-project/libhdfs3 master \n cmake_install\n}\n' scripts/setup-centos7.sh
-    sed -i '/^  run_and_time install_folly/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh
+    sed -i '/^  run_and_time install_protobuf/a \ \ run_and_time install_libhdfs3' scripts/setup-centos7.sh
     sed -i '/^dnf_install ccache/a\ \ yasm \\' scripts/setup-centos7.sh
   fi
   sed -i "s/yum -y install/sudo yum -y install/" ${VELOX_HOME}/scripts/setup-adapters.sh