Merge branch 'main' into wip_decimal_precision_loss

.asf.yaml

@@ -31,7 +31,14 @@ github:
     squash: true
     merge: false
     rebase: false
-  protected_branches: ~
+  protected_branches:
+    main:
+      required_pull_request_reviews:
+        dismiss_stale_reviews: true
+        required_approving_review_count: 1
+      required_signatures: false
+      required_linear_history: true
+      required_conversation_resolution: false
   features:
     issues: true
     discussions: true

.github/ISSUE_TEMPLATE/bug.yml

@@ -50,6 +50,8 @@ body:
       options:
         - Spark-3.2.x
         - Spark-3.3.x
+        - Spark-3.4.x
+        - Spark-3.5.x
     validations:
       required: false
 

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+blank_issues_enabled: false
+contact_links:
+  - name: Gluten Discussions
+    url: https://github.com/apache/incubator-gluten/discussions
+    about: Ask questions or discuss new feature ideas here.

.github/workflows/build_bundle_package.yml

@@ -38,7 +38,7 @@ on:
 jobs:
   build-native-lib:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_03_17
+    container: apache/gluten:gluten-vcpkg-builder_2024_05_29
     steps:
       - uses: actions/checkout@v2
       - name: Build Gluten velox third party
@@ -53,11 +53,17 @@ jobs:
           export NUM_THREADS=4
           ./dev/builddeps-veloxbe.sh --build_tests=OFF  --build_benchmarks=OFF --enable_s3=OFF \
           --enable_gcs=OFF --enable_hdfs=ON --enable_abfs=OFF
-      - uses: actions/upload-artifact@v2
+      - name: Upload native libs
+        uses: actions/upload-artifact@v2
         with:
           path: ./cpp/build/releases/
           name: velox-native-lib-${{github.sha}}
           retention-days: 1
+      - name: Upload Artifact Arrow Jar
+        uses: actions/upload-artifact@v2
+        with:
+          path: /root/.m2/repository/org/apache/arrow/
+          name: velox-arrow-jar-centos-7-${{github.sha}}
 
   build-bundle-package-ubuntu:
     if: startsWith(github.event.inputs.os, 'ubuntu')
@@ -71,6 +77,11 @@ jobs:
         with:
           name: velox-native-lib-${{github.sha}}
           path: ./cpp/build/releases
+      - name: Download All Arrow Jar Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: velox-arrow-jar-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
       - name: Setup java and maven
         run: |
           apt-get update && \
@@ -99,6 +110,11 @@ jobs:
         with:
           name: velox-native-lib-${{github.sha}}
           path: ./cpp/build/releases
+      - name: Download All Arrow Jar Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: velox-arrow-jar-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
       - name: Setup java and maven
         run: |
           yum update -y && yum install -y java-1.8.0-openjdk-devel wget && \
@@ -130,6 +146,11 @@ jobs:
         with:
           name: velox-native-lib-${{github.sha}}
           path: ./cpp/build/releases
+      - name: Download All Arrow Jar Artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: velox-arrow-jar-centos-7-${{github.sha}}
+          path: /root/.m2/repository/org/apache/arrow/
       - name: Update mirror list
         run: |
           sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true && \

.github/workflows/check_license.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: license header check
+on:
+  pull_request
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  license-check:
+    name: License Header Check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check License Header
+        run: |
+          git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}}
+          pip install regex
+          cd $GITHUB_WORKSPACE/
+          ./.github/workflows/util/check.sh ${{github.event.pull_request.base.sha}}

.github/workflows/code_style.yml

@@ -14,43 +14,56 @@
 # limitations under the License.
 
 name: Code style checks
-
 on:
-  pull_request
-
+  pull_request:
+    paths:
+      - '.github/workflows/code_style.yml'
+      - 'cpp/**'
+      - 'cpp-ch/**'
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
-  formatting-check:
-    name: Formatting Check
+  CPP-format-check:
+    name: CPP Format Check
     runs-on: ubuntu-latest
     strategy:
       matrix:
         path:
-          - check: 'cpp/core'
-            exclude: ''
-          - check: 'cpp/velox'
+          - check: 'cpp'
             exclude: ''
     steps:
       - uses: actions/checkout@v4
-      - name: Run clang-format style check for C/C++ programs.
+      - name: Run clang-format style check for C/C++
         uses: jidicula/[email protected]
         with:
           clang-format-version: '15'
           check-path: ${{ matrix.path['check'] }}
           fallback-style: 'Google' # optional
 
-  license-check:
-    name: License Header Check
+  CMake-format-check:
+    name: CMake Format Check
     runs-on: ubuntu-latest
-
+    container: ubuntu:22.04
     steps:
-      - uses: actions/checkout@v3
-
-      - name: Check License Header
+      - name: Install tools
+        run: |
+          apt update -y
+          apt install git python3-pip -y
+          pip3 install --user cmake-format
+      - uses: actions/checkout@v4
+      - name: Check CMake format
         run: |
-          git fetch --recurse-submodules=no origin main ${{github.event.pull_request.base.sha}}
-          pip install regex
-          dev/check.sh ${{github.event.pull_request.base.sha}}
+          git config --global --add safe.directory $GITHUB_WORKSPACE
+          cd $GITHUB_WORKSPACE/
+          fileList=$(find ./cpp ./cpp-ch -name CMakeLists.txt -o -name *.cmake)
+          for file in $fileList; do
+              /github/home/.local/bin/cmake-format --first-comment-is-literal True --in-place $file
+          done
+          if [ -n "$(git status --porcelain)" ]; then
+             echo "Please use cmake-format to format cmake files or apply the below patch."
+             git diff -- '*CMakeLists.txt' '*.cmake'
+             exit 1
+          fi
+          echo "No CMake format issue."

dev/check.sh → .github/workflows/util/check.sh

@@ -15,9 +15,9 @@
 # limitations under the License.
 
 export BASE_COMMIT=$1
-dev/check.py header branch
+./.github/workflows/util/check.py header branch
 if [ $? -ne 0 ]; then
-  dev/check.py header branch --fix
+  ./.github/workflows/util/check.py header branch --fix
   echo -e "\n==== Apply using:"
   echo "patch -p1 \<<EOF"
   git --no-pager diff

dev/excluded_copyright_files.txt → .github/workflows/util/excluded_copyright_files.txt

@@ -1,2 +1,3 @@
 *cpp-ch/local-engine/Storages/ch_parquet/arrow/*
 *dev/*
+*.github/workflows/util/*