Merge branch 'main' into dev4

apache · Jul 30, 2024 · 65305a8 · 65305a8
2 parents 946310f + 97f4eb2
commit 65305a8
Show file tree

Hide file tree

Showing 658 changed files with 40,048 additions and 11,469 deletions.
diff --git a/.github/workflows/build_bundle_package.yml b/.github/workflows/build_bundle_package.yml
@@ -15,6 +15,9 @@
 
 name: Build bundle package
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 concurrency:
   group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
   cancel-in-progress: true

diff --git a/.github/workflows/clickhouse_be_trigger.yml b/.github/workflows/clickhouse_be_trigger.yml
@@ -22,14 +22,13 @@ on:
       - '.github/workflows/clickhouse_be_trigger.yml'
       - 'pom.xml'
       - 'backends-clickhouse/**'
-      - 'gluten-celeborn/common'
-      - 'gluten-celeborn/package'
-      - 'gluten-celeborn/clickhouse'
+      - 'gluten-celeborn/common/**'
+      - 'gluten-celeborn/package/**'
+      - 'gluten-celeborn/clickhouse/**'
       - 'gluten-core/**'
       - 'gluten-ut/**'
       - 'shims/**'
       - 'tools/gluten-it/**'
-      - 'tools/gluten-te/**'
       - 'cpp-ch/**'
 
 jobs:

diff --git a/.github/workflows/velox_be.yml.deprecated b/.github/workflows/velox_be.yml.deprecated
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
@@ -33,7 +33,6 @@ on:
       - 'gluten-ut/**'
       - 'shims/**'
       - 'tools/gluten-it/**'
-      - 'tools/gluten-te/**'
       - 'ep/build-velox/**'
       - 'cpp/*'
       - 'cpp/CMake/**'
@@ -42,6 +41,7 @@ on:
       - 'dev/**'
 
 env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
   MVN_CMD: 'mvn -ntp'
 
 concurrency:
@@ -51,7 +51,7 @@ concurrency:
 jobs:
   build-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_05_29 # centos7 with dependencies installed
+    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -63,12 +63,13 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
-            ~/.m2/repository/org/apache/arrow/
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
-      - name: Build Gluten Velox third party
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
         if: ${{ steps.cache.outputs.cache-hit != 'true' }}
         run: |
-          source dev/ci-velox-buildstatic.sh
+          df -a
+          bash dev/ci-velox-buildstatic-centos-7.sh
       - name: Upload Artifact Native
         uses: actions/upload-artifact@v2
         with:
@@ -295,7 +296,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67,q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
             -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \
             -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \
@@ -307,7 +308,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q67 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
             -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \
             -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \
@@ -318,7 +319,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q95 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
             -d=OFFHEAP_SIZE:6g,spark.memory.offHeap.size=6g \
             -d=OFFHEAP_SIZE:4g,spark.memory.offHeap.size=4g \
@@ -329,7 +330,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
             -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
             -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \
@@ -340,7 +341,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q23a,q23b -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
             -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
             -d=FLUSH_MODE:DISABLED,spark.gluten.sql.columnar.backend.velox.flushablePartialAggregation=false,spark.gluten.sql.columnar.backend.velox.maxPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.maxExtendedPartialAggregationMemoryRatio=1.0,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinPct=100,spark.gluten.sql.columnar.backend.velox.abandonPartialAggregationMinRows=0 \
@@ -351,7 +352,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx3G sbin/gluten-it.sh parameterized \
             --local --preset=velox --benchmark-type=ds --error-on-memleak --queries=q97 -s=30.0 --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen -m=OffHeapExecutionMemory \
+            --data-gen-strategy=skip -m=OffHeapExecutionMemory \
             -d=ISOLATION:OFF,spark.gluten.memory.isolation=false \
             -d=ISOLATION:ON,spark.gluten.memory.isolation=true,spark.memory.storageFraction=0.1 \
             -d=OFFHEAP_SIZE:2g,spark.memory.offHeap.size=2g \
@@ -407,7 +408,7 @@ jobs:
           cd tools/gluten-it \
           && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries \
             --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0  --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \
-            --skip-data-gen  --random-kill-tasks --no-session-reuse
+            --data-gen-strategy=skip  --random-kill-tasks --no-session-reuse
 
   # run-tpc-test-ubuntu-sf30:
   #   needs: build-native-lib-centos-7
@@ -456,10 +457,10 @@ jobs:
   #         cd tools/gluten-it \
   #         && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \
   #           --local --preset=velox --benchmark-type=h --error-on-memleak -s=30.0  --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \
-  #           --skip-data-gen --shard=${{ matrix.shard }} \
+  #           --data-gen-strategy=skip --shard=${{ matrix.shard }} \
   #         && GLUTEN_IT_JVM_ARGS=-Xmx6G sbin/gluten-it.sh queries-compare \
   #           --local --preset=velox --benchmark-type=ds --error-on-memleak -s=30.0  --off-heap-size=8g --threads=12 --shuffle-partitions=72 --iterations=1 \
-  #           --skip-data-gen --shard=${{ matrix.shard }}
+  #           --data-gen-strategy=skip --shard=${{ matrix.shard }}
 
   run-tpc-test-centos8-uniffle:
     needs: build-native-lib-centos-7
@@ -532,7 +533,7 @@ jobs:
       fail-fast: false
       matrix:
         spark: [ "spark-3.2" ]
-        celeborn: [ "celeborn-0.5.0", "celeborn-0.4.1", "celeborn-0.3.2-incubating" ]
+        celeborn: [ "celeborn-0.5.1", "celeborn-0.4.2", "celeborn-0.3.2-incubating" ]
     runs-on: ubuntu-20.04
     container: ubuntu:22.04
     steps:
@@ -563,9 +564,9 @@ jobs:
       - name: TPC-H SF1.0 && TPC-DS SF1.0 Parquet local spark3.2 with ${{ matrix.celeborn }}
         run: |
           EXTRA_PROFILE=""
-          if [ "${{ matrix.celeborn }}" = "celeborn-0.4.1" ]; then
+          if [ "${{ matrix.celeborn }}" = "celeborn-0.4.2" ]; then
             EXTRA_PROFILE="-Pceleborn-0.4"
-          elif [ "${{ matrix.celeborn }}" = "celeborn-0.5.0" ]; then
+          elif [ "${{ matrix.celeborn }}" = "celeborn-0.5.1" ]; then
             EXTRA_PROFILE="-Pceleborn-0.5"
           fi
           echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
@@ -596,32 +597,15 @@ jobs:
         with:
           path: |
             ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
             /root/.m2/repository/org/apache/arrow/
           key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
-      - name: Update mirror list
-        run: |
-          sed -i -e "s|mirrorlist=|#mirrorlist=|g" /etc/yum.repos.d/CentOS-* || true
-          sed -i -e "s|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g" /etc/yum.repos.d/CentOS-* || true
-      - name: Setup build dependency
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: |
-          yum install sudo patch java-1.8.0-openjdk-devel wget -y
-          # Required by building arrow java.
-          wget https://downloads.apache.org/maven/maven-3/3.8.8/binaries/apache-maven-3.8.8-bin.tar.gz
-          tar -xvf apache-maven-3.8.8-bin.tar.gz && mv apache-maven-3.8.8 /usr/lib/maven
-          echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV
-      - name: Build Gluten Velox third party
-        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
-        run: |
-          source /opt/rh/gcc-toolset-9/enable
-          source ./dev/build_arrow.sh
-          install_arrow_deps
-          ./dev/builddeps-veloxbe.sh --run_setup_script=OFF --enable_ep_cache=OFF --build_tests=ON \
-              --build_examples=ON --build_benchmarks=ON --build_protobuf=ON
-      - name: Gluten CPP Test
+      - name: Build Gluten native libraries
+        if: steps.cache.outputs.cache-hit != 'true'
         run: |
-          cd ./cpp/build && \
-          ctest -V
+          df -a
+          bash dev/ci-velox-buildshared-centos-8.sh
       - uses: actions/upload-artifact@v2
         with:
           name: velox-native-lib-centos-8-${{github.sha}}

diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml
@@ -20,14 +20,17 @@ on:
     branches:
       - 'main'
 
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+
 concurrency:
   group: ${{ github.repository }}-${{ github.workflow }}
   cancel-in-progress: false
 
 jobs:
-  cache-native-lib:
+  cache-native-lib-centos-7:
     runs-on: ubuntu-20.04
-    container: apache/gluten:gluten-vcpkg-builder_2024_05_29 # centos7 with dependencies installed
+    container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed
     steps:
       - uses: actions/checkout@v2
       - name: Generate cache key
@@ -41,11 +44,12 @@ jobs:
           path: |
             ./cpp/build/releases/
             /root/.m2/repository/org/apache/arrow/
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
-      - name: Build Gluten Velox third party
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
         if: steps.check-cache.outputs.cache-hit != 'true'
         run: |
-          source dev/ci-velox-buildstatic.sh
+          df -a
+          bash dev/ci-velox-buildstatic-centos-7.sh
       - name: Cache
         if: steps.check-cache.outputs.cache-hit != 'true'
         id: cache
@@ -54,7 +58,43 @@ jobs:
           path: |
             ./cpp/build/releases/
             /root/.m2/repository/org/apache/arrow/  
-          key: cache-velox-build-${{ hashFiles('./cache-key') }}
+          key: cache-velox-build-centos-7-${{ hashFiles('./cache-key') }}
+
+  cache-native-lib-centos-8:
+    runs-on: ubuntu-20.04
+    container: ghcr.io/facebookincubator/velox-dev:centos8
+    steps:
+      - uses: actions/checkout@v2
+      - name: Generate cache key
+        run: |
+          echo ${{ hashFiles('./ep/build-velox/src/**', './dev/**', './cpp/*', './github/workflows/*') }} > cache-key
+      - name: Check existing caches
+        id: check-cache
+        uses: actions/cache/restore@v3
+        with:
+          lookup-only: true
+          path: |
+            ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
+      - name: Build Gluten native libraries
+        if: steps.check-cache.outputs.cache-hit != 'true'
+        run: |
+          df -a
+          bash dev/ci-velox-buildshared-centos-8.sh
+      - name: Cache
+        if: steps.check-cache.outputs.cache-hit != 'true'
+        id: cache
+        uses: actions/cache/save@v3
+        with:
+          path: |
+            ./cpp/build/releases/
+            ./cpp/build/velox/udf/examples/
+            ./cpp/build/velox/benchmarks/
+            /root/.m2/repository/org/apache/arrow/
+          key: cache-velox-build-centos-8-${{ hashFiles('./cache-key') }}
 
   # ccache-native-lib-ubuntu-velox-ut:
   #   runs-on: ubuntu-20.04
@@ -72,7 +112,7 @@ jobs:
   #       working-directory: ${{ github.workspace }}
   #       run: |
   #         mkdir -p '${{ env.CCACHE_DIR }}'
-  #     - name: Build Gluten velox third party
+  #     - name: Build Gluten native libraries
   #       run: |
   #         rm -rf /opt/miniconda-for-velox/
   #         cd ep/build-velox/src && \
@@ -110,7 +150,7 @@ jobs:
 #        working-directory: ${{ github.workspace }}
 #        run: |
 #          mkdir -p '${{ env.CCACHE_DIR }}'
-#      - name: Build Gluten velox third party
+#      - name: Build Gluten native libraries
 #        run: |
 #          rm -rf /opt/miniconda-for-velox/
 #          cd ep/build-velox/src && \
@@ -126,4 +166,4 @@ jobs:
 #      - uses: actions/cache/save@v3
 #        with:
 #          path: '${{ env.CCACHE_DIR }}'
-#          key: ccache-centos-release-default
+#          key: ccache-centos-release-default
diff --git a/.github/workflows/velox_tpch_bench.yml b/.github/workflows/velox_tpch_bench.yml