Merge branch 'main' into gluten_6977

.github/workflows/build_bundle_package.yml

@@ -52,7 +52,6 @@ jobs:
           source /opt/rh/devtoolset-9/enable && \
           source /opt/gluten//dev/vcpkg/env.sh && \
           cd $GITHUB_WORKSPACE/ && \
-          sed -i '/^headers/d' ep/build-velox/build/velox_ep/CMakeLists.txt && \
           export NUM_THREADS=4
           ./dev/builddeps-veloxbe.sh --build_tests=OFF  --build_benchmarks=OFF --enable_s3=OFF \
           --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF

.github/workflows/clickhouse_be_trigger.yml

@@ -34,17 +34,21 @@ on:
 jobs:
   add-comment:
     runs-on: ubuntu-latest
+    permissions: write-all
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
       - name: Sleep for Dev PR workflow done
         run: |
           sleep 15
-        
       - name: Add comment to PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          COMMENT="Run Gluten Clickhouse CI"
-          URL=$(jq -r .pull_request.comments_url "$GITHUB_EVENT_PATH")
-          curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST -d "{\"body\":\"$COMMENT\"}" "${URL}"
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.createComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.payload.number,
+              body: "Run Gluten Clickhouse CI"
+            });

.github/workflows/dev_cron.yml

@@ -27,15 +27,16 @@ jobs:
   process:
     name: Process
     runs-on: ubuntu-latest
+    permissions: write-all
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Comment Issues link
         if: |
           github.event_name == 'pull_request_target' &&
             (github.event.action == 'opened' ||
              github.event.action == 'edited')
-        uses: actions/github-script@v3
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |
@@ -47,7 +48,7 @@ jobs:
           github.event_name == 'pull_request_target' &&
             (github.event.action == 'opened' ||
              github.event.action == 'edited')
-        uses: actions/github-script@v3
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/dev_cron/issues_link.js

@@ -35,7 +35,7 @@ async function haveComment(github, context, pullRequestNumber, body) {
     page: 1
   };
   while (true) {
-    const response = await github.issues.listComments(options);
+    const response = await github.rest.issues.listComments(options);
     if (response.data.some(comment => comment.body === body)) {
       return true;
     }
@@ -52,7 +52,7 @@ async function commentISSUESURL(github, context, pullRequestNumber, issuesID) {
   if (await haveComment(github, context, pullRequestNumber, issuesURL)) {
     return;
   }
-  await github.issues.createComment({
+  await github.rest.issues.createComment({
     owner: context.repo.owner,
     repo: context.repo.repo,
     issue_number: pullRequestNumber,

.github/workflows/dev_cron/title_check.js

@@ -25,7 +25,7 @@ function haveISSUESID(title) {
 }
 
 async function commentOpenISSUESIssue(github, context, pullRequestNumber) {
-  const {data: comments} = await github.issues.listComments({
+  const {data: comments} = await github.rest.issues.listComments({
     owner: context.repo.owner,
     repo: context.repo.repo,
     issue_number: pullRequestNumber,
@@ -36,7 +36,7 @@ async function commentOpenISSUESIssue(github, context, pullRequestNumber) {
   }
   const commentPath = ".github/workflows/dev_cron/title_check.md";
   const comment = fs.readFileSync(commentPath).toString();
-  await github.issues.createComment({
+  await github.rest.issues.createComment({
     owner: context.repo.owner,
     repo: context.repo.repo,
     issue_number: pullRequestNumber,

.github/workflows/docker_image.yml

@@ -0,0 +1,50 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Build and Push Docker Image
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - '.github/workflows/docker_image.yml'
+  schedule:
+    - cron: '0 20 * * 0'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          file: dev/vcpkg/docker/Dockerfile.gha
+          push: true
+          tags: apache/gluten:vcpkg-centos-7

.github/workflows/util/install_spark_resources.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Download Spark resources, required by some Spark UTs. The resource path should be set
+# for spark.test.home in mvn test.
+
+set -e
+
+INSTALL_DIR=$GITHUB_WORKSPACE
+case "$1" in
+3.2)
+    # Spark-3.2
+    cd ${INSTALL_DIR} && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.2.2/spark-3.2.2-bin-hadoop3.2.tgz && \
+    tar --strip-components=1 -xf spark-3.2.2-bin-hadoop3.2.tgz spark-3.2.2-bin-hadoop3.2/jars/ && \
+    rm -rf spark-3.2.2-bin-hadoop3.2.tgz && \
+    mkdir -p ${INSTALL_DIR}/shims/spark32/spark_home/assembly/target/scala-2.12 && \
+    mv jars ${INSTALL_DIR}/shims/spark32/spark_home/assembly/target/scala-2.12 && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.2.2.tar.gz && \
+    tar --strip-components=1 -xf v3.2.2.tar.gz spark-3.2.2/sql/core/src/test/resources/  && \
+    mkdir -p shims/spark32/spark_home/ && \
+    mv sql shims/spark32/spark_home/
+    ;;
+3.3)
+    # Spark-3.3
+    cd ${INSTALL_DIR} && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.3.1-bin-hadoop3.tgz spark-3.3.1-bin-hadoop3/jars/ && \
+    rm -rf spark-3.3.1-bin-hadoop3.tgz && \
+    mkdir -p ${INSTALL_DIR}/shims/spark33/spark_home/assembly/target/scala-2.12 && \
+    mv jars ${INSTALL_DIR}/shims/spark33/spark_home/assembly/target/scala-2.12 && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.3.1.tar.gz && \
+    tar --strip-components=1 -xf v3.3.1.tar.gz spark-3.3.1/sql/core/src/test/resources/  && \
+    mkdir -p shims/spark33/spark_home/ && \
+    mv sql shims/spark33/spark_home/
+    ;;
+3.4)
+    # Spark-3.4
+    cd ${INSTALL_DIR} && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.4.2-bin-hadoop3.tgz spark-3.4.2-bin-hadoop3/jars/ && \
+    rm -rf spark-3.4.2-bin-hadoop3.tgz && \
+    mkdir -p ${INSTALL_DIR}/shims/spark34/spark_home/assembly/target/scala-2.12 && \
+    mv jars ${INSTALL_DIR}/shims/spark34/spark_home/assembly/target/scala-2.12 && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.4.2.tar.gz && \
+    tar --strip-components=1 -xf v3.4.2.tar.gz spark-3.4.2/sql/core/src/test/resources/  && \
+    mkdir -p shims/spark34/spark_home/ && \
+    mv sql shims/spark34/spark_home/
+    ;;
+3.5)
+    # Spark-3.5
+    cd ${INSTALL_DIR} && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.5.1-bin-hadoop3.tgz spark-3.5.1-bin-hadoop3/jars/ && \
+    rm -rf spark-3.5.1-bin-hadoop3.tgz && \
+    mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
+    mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \
+    tar --strip-components=1 -xf v3.5.1.tar.gz spark-3.5.1/sql/core/src/test/resources/  && \
+    mkdir -p shims/spark35/spark_home/ && \
+    mv sql shims/spark35/spark_home/
+    ;;
+3.5-scala2.13)
+    # Spark-3.5, scala 2.13
+    cd ${INSTALL_DIR} && \
+    wget -nv https://archive.apache.org/dist/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz && \
+    tar --strip-components=1 -xf spark-3.5.1-bin-hadoop3.tgz spark-3.5.1-bin-hadoop3/jars/ && \
+    rm -rf spark-3.5.1-bin-hadoop3.tgz && \
+    mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
+    mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \
+    wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.1.tar.gz && \
+    tar --strip-components=1 -xf v3.5.1.tar.gz spark-3.5.1/sql/core/src/test/resources/  && \
+    mkdir -p shims/spark35/spark_home/ && \
+    mv sql shims/spark35/spark_home/
+    ;;
+*)
+    echo "Spark version is expected to be specified."
+    exit 1
+    ;;
+esac

.github/workflows/util/setup_helper.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+function install_maven {
+  (
+    cd /opt/
+    wget -nv https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
+    tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven
+  )
+  echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
+}
+
+for cmd in "$@"
+do
+    echo "Running: $cmd"
+    "$cmd"
+done