Merge branch 'main' into spill

.github/workflows/clickhouse_be_trigger.yml

@@ -29,7 +29,6 @@ on:
       - 'gluten-ut/**'
       - 'shims/**'
       - 'tools/gluten-it/**'
-      - 'tools/gluten-te/**'
       - 'cpp-ch/**'
 
 jobs:

.github/workflows/velox_docker.yml

@@ -33,7 +33,6 @@ on:
       - 'gluten-ut/**'
       - 'shims/**'
       - 'tools/gluten-it/**'
-      - 'tools/gluten-te/**'
       - 'ep/build-velox/**'
       - 'cpp/*'
       - 'cpp/CMake/**'
@@ -64,13 +63,13 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            ~/.m2/repository/org/apache/arrow/
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
-      - name: Build Gluten Velox third party
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
         run: |
           df -a
-          source dev/ci-velox-buildstatic.sh
+          bash dev/ci-velox-buildstatic-centos-7.sh
       - name: Upload Artifact Native
         uses: actions/upload-artifact@v2
         with:
@@ -598,32 +597,15 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
             /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
-      - name: Update mirror list
-        run: |
-          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
-          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
-      - name: Setup build dependency
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
+      - name: Build Gluten native libraries
+        if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          # Required by building arrow java.
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
-          tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven
-          echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
-      - name: Build Gluten Velox third party
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: |
-          source /opt/rh/gcc-toolset-9/enable
-          source ./dev/build_arrow.sh
-          install_arrow_deps
-          ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \
-              --build_examples=ON --build_benchmarks=ON --build_protobuf=ON
-      - name: Gluten CPP Test
-        run: |
-          cd ./cpp/build && \
-          ctest -V
+          df -a
+          bash dev/ci-velox-buildshared-centos-8.sh
       - uses: actions/upload-artifact@v2
         with:
           name: velox-native-lib-centos-8-${{github.sha}}

.github/workflows/velox_docker_cache.yml

@@ -28,9 +28,9 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
-  cache-native-lib:
+  cache-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_05_29 # centos7 with dependencies installed
+    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -44,11 +44,12 @@ jobs:
           path: |
             ./cpp/build/releases/
             /root/.m2/repository/org/apache/arrow/
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
-      - name: Build Gluten Velox third party
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
         if: steps.check-cache.outputs.cache-hit != 'true'
         run: |
-          source dev/ci-velox-buildstatic.sh
+          df -a
+          bash dev/ci-velox-buildstatic-centos-7.sh
       - name: Cache
         if: steps.check-cache.outputs.cache-hit != 'true'
         id: cache
@@ -57,7 +58,43 @@ jobs:
           path: |
             ./cpp/build/releases/
             /root/.m2/repository/org/apache/arrow/  
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+
+  cache-native-lib-centos-8:
+    runs-on: ubuntu-20.04
+    container: ghcr.io/facebookincubator/velox-dev:centos8
+    steps:
+      - uses: actions/checkout@v2
+      - name: Generate cache key
+        run: |
+          echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key
+      - name: Check existing caches
+        id: check-cache
+        uses: actions/cache/restore@v3
+        with:
+          lookup-only: true
+          path: |
+            ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
+        if: steps.check-cache.outputs.cache-hit != 'true'
+        run: |
+          df -a
+          bash dev/ci-velox-buildshared-centos-8.sh
+      - name: Cache
+        if: steps.check-cache.outputs.cache-hit != 'true'
+        id: cache
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
 
   # ccache-native-lib-ubuntu-velox-ut:
   #   runs-on: ubuntu-20.04
@@ -75,7 +112,7 @@ jobs:
   #       working-directory: ${{ github.workspace }}
   #       run: |
   #         mkdir -p '${{ env.CCACHE_DIR }}'
-  #     - name: Build Gluten velox third party
+  #     - name: Build Gluten native libraries
   #       run: |
   #         rm -rf /opt/miniconda-for-velox/
   #         cd ep/build-velox/src && \
@@ -113,7 +150,7 @@ jobs:
 #        working-directory: ${{ github.workspace }}
 #        run: |
 #          mkdir -p '${{ env.CCACHE_DIR }}'
-#      - name: Build Gluten velox third party
+#      - name: Build Gluten native libraries
 #        run: |
 #          rm -rf /opt/miniconda-for-velox/
 #          cd ep/build-velox/src && \

.github/workflows/velox_nightly.yml.disabled → .github/workflows/velox_weekly.yml

@@ -13,14 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-name: Velox backend nightly job
+name: Velox backend weekly job
 
 on:
   pull_request:
     paths:
-      - '.github/workflows/velox_nightly.yml'
+      - '.github/workflows/velox_weekly.yml'
   schedule:
-    - cron: '0 20 * * *'
+    - cron: '0 20 * * 0'
+
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
@@ -37,25 +40,28 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Update mirror list
-        if: matrix.os == 'centos:8'
         run: |
           sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
           sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
       - name: build
         run: |
           yum update -y
           yum install -y epel-release sudo dnf
-          if [ "${{ matrix.os }}" = "centos:8" ]; then
+          if [ "${{ matrix.os }}" = "centos:7" ]; then
+            yum install -y centos-release-scl
+            rm /etc/yum.repos.d/CentOS-SCLo-scl.repo -f
+            sed -i \
+            -e 's/^mirrorlist/#mirrorlist/' \
+            -e 's/^#baseurl/baseurl/' \
+            -e 's/mirror\.centos\.org/vault.centos.org/' \
+            /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
+          else
             dnf install -y --setopt=install_weak_deps=False gcc-toolset-9
             source /opt/rh/gcc-toolset-9/enable || exit 1
-          else
-            yum install -y centos-release-scl
-            yum install -y devtoolset-9
-            source /opt/rh/devtoolset-9/enable || exit 1
           fi
-          yum install -y java-1.8.0-openjdk-devel patch wget git && \
+          yum install -y java-1.8.0-openjdk-devel patch wget git
           export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \
-          export PATH=$JAVA_HOME/bin:$PATH && \
+          export PATH=$JAVA_HOME/bin:$PATH
           wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz && \
           tar -xvf apache-maven-3.8.8-bin.tar.gz && \
           mv apache-maven-3.8.8 /usr/lib/maven && \
@@ -76,7 +82,8 @@ jobs:
       - name: build
         run: |
           # To avoid the prompt for region selection during installing tzdata.
-          export DEBIAN_FRONTEND="noninteractive"
-          apt-get update && apt-get install -y sudo openjdk-8-jdk maven wget git
+          export DEBIAN_FRONTEND=noninteractive
+          apt-get update && apt-get install -y sudo maven wget git
+          sudo apt-get install -y openjdk-8-jdk
           export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
           cd $GITHUB_WORKSPACE/ && ./dev/package.sh

.gitignore

@@ -58,6 +58,7 @@ build/
 Testing/
 cmake-build-debug/
 cmake-build-release/
+ep/_ep/
 
 # Editor temporary/working/backup files #
 .#*

DISCLAIMER-WIP

@@ -0,0 +1,18 @@
+Apache Gluten (Incubating) is an effort undergoing incubation at the Apache
+Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+
+Incubation is required of all newly accepted projects until a further review
+indicates that the infrastructure, communications, and decision making process
+have stabilized in a manner consistent with other successful ASF projects.
+
+While incubation status is not necessarily a reflection of the completeness
+or stability of the code, it does indicate that the project has yet to be
+fully endorsed by the ASF.
+
+Some of the incubating project’s releases may not be fully compliant with ASF policy.
+For example, releases may have incomplete or un-reviewed licensing conditions.
+What follows is a list of issues the project is currently aware of (this list is likely to be incomplete):
+
+* Releases may have incomplete licensing conditions.
+
+If you are planning to incorporate this work into your product/project, please be aware that you will need to conduct a thorough licensing review to determine the overall implications of including this work. For the current status of this project through the Apache Incubator, visit: https://incubator.apache.org/projects/gluten.html