Merge branch 'main' of https://github.com/gaoyangxiaozhu/gluten into …

…gayangya/split_refactor

backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala

@@ -1017,7 +1017,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
     }
   }
 
-  test("test explode/posexplode function") {
+  ignore("test explode/posexplode function") {
     Seq("explode", "posexplode").foreach {
       func =>
         // Literal: func(literal)
@@ -1190,7 +1190,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
                           |""".stripMargin)(_)
   }
 
-  test("test multi-generate") {
+  ignore("test multi-generate") {
     withTable("t") {
       sql("CREATE TABLE t (col1 array<struct<a int, b string>>, col2 array<int>) using parquet")
       sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')), array(1, 2))")
@@ -1588,7 +1588,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
     }
   }
 
-  test("test array literal") {
+  ignore("test array literal") {
     withTable("array_table") {
       sql("create table array_table(a array<bigint>) using parquet")
       sql("insert into table array_table select array(1)")
@@ -1601,7 +1601,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
     }
   }
 
-  test("test map literal") {
+  ignore("test map literal") {
     withTable("map_table") {
       sql("create table map_table(a map<bigint, string>) using parquet")
       sql("insert into table map_table select map(1, 'hello')")

cpp-ch/local-engine/Functions/SparkFunctionHashingExtended.h

@@ -28,6 +28,7 @@
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/NumberTraits.h>
 #include <Functions/FunctionsHashing.h>
+#include <Common/NaNUtils.h>
 
 namespace DB
 {
@@ -200,14 +201,14 @@ class SparkFunctionAnyHash : public IFunction
         {
             if constexpr (std::is_same_v<T, Float32>)
             {
-                if (n == -0.0f) [[unlikely]]
+                if (n == -0.0f || isNaN(n)) [[unlikely]]
                     return applyNumber<Int32>(0, seed);
                 else
                     return Impl::apply(reinterpret_cast<const char *>(&n), sizeof(n), seed);
             }
             else
             {
-                if (n == -0.0) [[unlikely]]
+                if (n == -0.0 || isNaN(n)) [[unlikely]]
                     return applyNumber<Int64>(0, seed);
                 else
                     return Impl::apply(reinterpret_cast<const char *>(&n), sizeof(n), seed);

cpp/CMakeLists.txt

@@ -68,9 +68,13 @@ if(NOT DEFINED VELOX_HOME)
 endif()
 
 if(${CMAKE_BUILD_TYPE} STREQUAL "Debug")
-  set(ARROW_HOME ${VELOX_HOME}/_build/debug/third_party/arrow_ep)
+  set(ARROW_HOME
+      ${VELOX_HOME}/_build/debug/CMake/resolve_dependency_modules/arrow/arrow_ep/
+  )
 else()
-  set(ARROW_HOME ${VELOX_HOME}/_build/release/third_party/arrow_ep)
+  set(ARROW_HOME
+      ${VELOX_HOME}/_build/release/CMake/resolve_dependency_modules/arrow/arrow_ep
+  )
 endif()
 
 include(ResolveDependency)

cpp/core/config/GlutenConfig.h

@@ -30,8 +30,6 @@ const std::string kGlutenSaveDir = "spark.gluten.saveDir";
 
 const std::string kCaseSensitive = "spark.sql.caseSensitive";
 
-const std::string kLegacySize = "spark.sql.legacy.sizeOfNull";
-
 const std::string kSessionTimezone = "spark.sql.session.timeZone";
 
 const std::string kIgnoreMissingFiles = "spark.sql.files.ignoreMissingFiles";

cpp/velox/compute/WholeStageResultIterator.cc

@@ -454,8 +454,6 @@ std::unordered_map<std::string, std::string> WholeStageResultIterator::getQueryC
     }
     // Adjust timestamp according to the above configured session timezone.
     configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true";
-    // Align Velox size function with Spark.
-    configs[velox::core::QueryConfig::kSparkLegacySizeOfNull] = std::to_string(veloxCfg_->get<bool>(kLegacySize, true));
 
     {
       // partial aggregation memory config

docs/get-started/ClickHouse.md

@@ -629,19 +629,26 @@ public read-only account：gluten/hN2xX3uQ4m
 
 ### Celeborn support
 
-Gluten with clickhouse backend has not yet supportted [Celeborn](https://github.com/apache/celeborn) natively as remote shuffle service using columar shuffle. However, you can still use Celeborn with row shuffle, which means a ColumarBatch will be converted to a row during shuffle.
-Below introduction is used to enable this feature:
+Gluten with clickhouse backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`.
+
+Below introduction is used to enable this feature.
 
 First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster.
 
+When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows:
+
+```
+mvn clean package -Pbackends-clickhouse -Pspark-3.3 -Pceleborn -DskipTests
+```
+
 Then add the Spark Celeborn Client packages to your Spark application's classpath(usually add them into `$SPARK_HOME/jars`).
 
 - Celeborn: celeborn-client-spark-3-shaded_2.12-[celebornVersion].jar
 
-Currently to use Celeborn following configurations are required in `spark-defaults.conf`
+Currently to use Gluten following configurations are required in `spark-defaults.conf`
 
 ```
-spark.shuffle.manager org.apache.spark.shuffle.celeborn.SparkShuffleManager
+spark.shuffle.manager org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager
 
 # celeborn master
 spark.celeborn.master.endpoints clb-master:9097
@@ -670,24 +677,6 @@ spark.celeborn.storage.hdfs.dir hdfs://<namenode>/celeborn
 spark.dynamicAllocation.enabled false
 ```
 
-#### Celeborn Columnar Shuffle Support
-Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`.
-The native Celeborn support can be enabled by the following configuration
-```
-spark.shuffle.manager=org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleManager
-```
-
-quickly start a celeborn cluster
-```shell
-wget https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz && \
-tar -zxvf apache-celeborn-0.3.2-incubating-bin.tgz && \
-mv apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
-mv apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml && \
-mkdir /opt/hadoop && chmod 777 /opt/hadoop && \
-echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
-bash apache-celeborn-0.3.2-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.2-incubating-bin/sbin/start-worker.sh
-```
-
 ### Columnar shuffle mode
 We have two modes of columnar shuffle   
 1. prefer cache

docs/get-started/Velox.md

@@ -224,11 +224,11 @@ Currently there are several ways to asscess S3 in Spark. Please refer [Velox S3]
 
 Gluten with velox backend supports [Celeborn](https://github.com/apache/celeborn) as remote shuffle service. Currently, the supported Celeborn versions are `0.3.x` and `0.4.0`.
 
-Below introduction is used to enable this feature
+Below introduction is used to enable this feature.
 
 First refer to this URL(https://github.com/apache/celeborn) to setup a celeborn cluster.
 
-When compiling the Gluten Java module, it's required to enable `rss` profile, as follows:
+When compiling the Gluten Java module, it's required to enable `celeborn` profile, as follows:
 
 ```
 mvn clean package -Pbackends-velox -Pspark-3.3 -Pceleborn -DskipTests

docs/get-started/build-guide.md

@@ -55,17 +55,18 @@ Please set them via `--`, e.g., `--velox_home=/YOUR/PATH`.
 ### Maven build parameters
 The below parameters can be set via `-P` for mvn.
 
-| Parameters          | Description                                                                  | Default state |
-|---------------------|------------------------------------------------------------------------------|---------------|
-| backends-velox      | Build Gluten Velox backend.                                                  | disabled       |
-| backends-clickhouse | Build Gluten ClickHouse backend.                                             | disabled      |
-| rss                 | Build Gluten with Remote Shuffle Service, only applicable for Velox backend. | disabled       |
-| delta               | Build Gluten with Delta Lake support.                                        | disabled      |
-| iceberg             | Build Gluten with Iceberg support.                                           | disabled      |
-| spark-3.2           | Build Gluten for Spark 3.2.                                                  | enabled       |
-| spark-3.3           | Build Gluten for Spark 3.3.                                                  | disabled       |
-| spark-3.4           | Build Gluten for Spark 3.4.                                                  | disabled       |
-| spark-3.5           | Build Gluten for Spark 3.5.                                                  | disabled       |
+| Parameters          | Description                           | Default state |
+|---------------------|---------------------------------------|---------------|
+| backends-velox      | Build Gluten Velox backend.           | disabled      |
+| backends-clickhouse | Build Gluten ClickHouse backend.      | disabled      |
+| celeborn            | Build Gluten with Celeborn.           | disabled      |
+| uniffle             | Build Gluten with Uniffle.            | disabled      |
+| delta               | Build Gluten with Delta Lake support. | disabled      |
+| iceberg             | Build Gluten with Iceberg support.    | disabled      |
+| spark-3.2           | Build Gluten for Spark 3.2.           | enabled       |
+| spark-3.3           | Build Gluten for Spark 3.3.           | disabled      |
+| spark-3.4           | Build Gluten for Spark 3.4.           | disabled      |
+| spark-3.5           | Build Gluten for Spark 3.5.           | disabled      |
 
 ## Gluten Jar for Deployment
 The gluten jar built out is under `GLUTEN_SRC/package/target/`.

ep/build-velox/src/build_velox.sh

@@ -147,6 +147,8 @@ function compile {
   echo "NUM_THREADS_OPTS: $NUM_THREADS_OPTS"
 
   export simdjson_SOURCE=AUTO
+  # Quick fix for CI error due to velox rebase
+  export Arrow_SOURCE=BUNDLED
   if [ $ARCH == 'x86_64' ]; then
     make $COMPILE_TYPE $NUM_THREADS_OPTS EXTRA_CMAKE_FLAGS="${COMPILE_OPTION}"
   elif [[ "$ARCH" == 'arm64' || "$ARCH" == 'aarch64' ]]; then

ep/build-velox/src/get_velox.sh

@@ -256,11 +256,11 @@ function apply_compilation_fixes {
   current_dir=$1
   velox_home=$2
   sudo cp ${current_dir}/modify_velox.patch ${velox_home}/
-  sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/third_party/
-  sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch ${velox_home}/third_party/
+  sudo cp ${current_dir}/modify_arrow.patch ${velox_home}/CMake/resolve_dependency_modules/arrow/
+  sudo cp ${current_dir}/modify_arrow_dataset_scan_option.patch ${velox_home}/CMake/resolve_dependency_modules/arrow/
   git add ${velox_home}/modify_velox.patch # to avoid the file from being deleted by git clean -dffx :/
-  git add ${velox_home}/third_party/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/
-  git add ${velox_home}/third_party/modify_arrow_dataset_scan_option.patch # to avoid the file from being deleted by git clean -dffx :/
+  git add ${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow.patch # to avoid the file from being deleted by git clean -dffx :/
+  git add ${velox_home}/CMake/resolve_dependency_modules/arrow/modify_arrow_dataset_scan_option.patch # to avoid the file from being deleted by git clean -dffx :/
   cd ${velox_home}
   echo "Applying patch to Velox source code..."
   git apply modify_velox.patch

ep/build-velox/src/modify_velox.patch

@@ -35,8 +35,31 @@ index d49115f12..1aaa8e532 100644
 +          IMPORTED_LOCATION_DEBUG "${LZ4_LIBRARY_DEBUG}")
 +  endif()
  endif()
+diff --git a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt
+index 3f01df2fd..8c1c493f3 100644
+--- a/CMake/resolve_dependency_modules/arrow/CMakeLists.txt
++++ b/CMake/resolve_dependency_modules/arrow/CMakeLists.txt
+@@ -24,6 +24,9 @@ if(VELOX_ENABLE_ARROW)
+   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
+   set(ARROW_CMAKE_ARGS
+       -DARROW_PARQUET=OFF
++      -DARROW_PARQUET=ON
++      -DARROW_FILESYSTEM=ON
++      -DARROW_PROTOBUF_USE_SHARED=OFF
+       -DARROW_WITH_THRIFT=ON
+       -DARROW_WITH_LZ4=ON
+       -DARROW_WITH_SNAPPY=ON
+@@ -66,6 +69,8 @@ if(VELOX_ENABLE_ARROW)
+     arrow_ep
+     PREFIX ${ARROW_PREFIX}
+     URL ${VELOX_ARROW_SOURCE_URL}
++    PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch
++    COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch
+     URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM}
+     SOURCE_SUBDIR cpp
+     CMAKE_ARGS ${ARROW_CMAKE_ARGS}
 diff --git a/CMakeLists.txt b/CMakeLists.txt
-index 5c7bf770a..9f897f577 100644
+index bb7c49907..3372d48b4 100644
 --- a/CMakeLists.txt
 +++ b/CMakeLists.txt
 @@ -234,10 +234,15 @@ if(VELOX_ENABLE_ABFS)
@@ -59,7 +82,7 @@ index 5c7bf770a..9f897f577 100644
    add_definitions(-DVELOX_ENABLE_HDFS3)
  endif()
 
-@@ -377,7 +382,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
+@@ -378,7 +383,7 @@ resolve_dependency(Boost 1.77.0 COMPONENTS ${BOOST_INCLUDE_LIBRARIES})
  # for reference. find_package(range-v3)
 
  set_source(gflags)
@@ -68,31 +91,6 @@ index 5c7bf770a..9f897f577 100644
  if(NOT TARGET gflags::gflags)
    # This is a bit convoluted, but we want to be able to use gflags::gflags as a
    # target even when velox is built as a subproject which uses
-
-diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
-index ce4c24dbe..785a2acc6 100644
---- a/third_party/CMakeLists.txt
-+++ b/third_party/CMakeLists.txt
-@@ -26,7 +26,9 @@ if(VELOX_ENABLE_ARROW)
-   endif()
-   set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep")
-   set(ARROW_CMAKE_ARGS
--      -DARROW_PARQUET=OFF
-+      -DARROW_PARQUET=ON
-+      -DARROW_FILESYSTEM=ON
-+      -DARROW_PROTOBUF_USE_SHARED=OFF
-       -DARROW_WITH_THRIFT=ON
-       -DARROW_WITH_LZ4=ON
-       -DARROW_WITH_SNAPPY=ON
-@@ -69,6 +71,8 @@ if(VELOX_ENABLE_ARROW)
-     arrow_ep
-     PREFIX ${ARROW_PREFIX}
-     URL ${VELOX_ARROW_SOURCE_URL}
-+    PATCH_COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow.patch
-+    COMMAND patch -p1 < ${CMAKE_CURRENT_SOURCE_DIR}/modify_arrow_dataset_scan_option.patch
-     URL_HASH ${VELOX_ARROW_BUILD_SHA256_CHECKSUM}
-     SOURCE_SUBDIR cpp
-     CMAKE_ARGS ${ARROW_CMAKE_ARGS}
 diff --git a/velox/common/process/tests/CMakeLists.txt b/velox/common/process/tests/CMakeLists.txt
 index 6797697a1..3e241f8f7 100644
 --- a/velox/common/process/tests/CMakeLists.txt