diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml index ee6bf9797d41b..f3066b7baae2d 100644 --- a/.github/workflows/velox_docker.yml +++ b/.github/workflows/velox_docker.yml @@ -51,7 +51,7 @@ concurrency: jobs: build-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key diff --git a/.github/workflows/velox_docker_cache.yml b/.github/workflows/velox_docker_cache.yml index 0fa771de07ee8..3fc0fc50f8473 100644 --- a/.github/workflows/velox_docker_cache.yml +++ b/.github/workflows/velox_docker_cache.yml @@ -30,7 +30,7 @@ concurrency: jobs: cache-native-lib-centos-7: runs-on: ubuntu-20.04 - container: apache/gluten:gluten-vcpkg-builder_2024_07_11 # centos7 with dependencies installed + container: apache/gluten:gluten-vcpkg-builder_2024_08_05 # centos7 with dependencies installed steps: - uses: actions/checkout@v2 - name: Generate cache key diff --git a/backends-clickhouse/pom.xml b/backends-clickhouse/pom.xml index f2ec45a5169a4..9c2103003d83a 100644 --- a/backends-clickhouse/pom.xml +++ b/backends-clickhouse/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT 4.0.0 diff --git a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala index d903304367d06..0efc1414ce338 100644 --- a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala @@ -2435,6 +2435,16 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends GlutenClickHouseTPCHAbstr } } + test("GLUTEN-6669: test cast string to boolean") { + withSQLConf( + SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> (ConstantFolding.ruleName + "," + NullPropagation.ruleName)) { + runQueryAndCompare( + "select cast('1' as boolean), cast('t' as boolean), cast('all' as boolean), cast('f' as boolean)", + noFallBack = false + )(checkGlutenOperatorMatch[ProjectExecTransformer]) + } + } + test("GLUTEN-4032: fix shuffle read coredump after union") { val sql = """ diff --git a/backends-velox/pom.xml b/backends-velox/pom.xml index de529a34df2fc..0fe8f5f6fd8e3 100755 --- a/backends-velox/pom.xml +++ b/backends-velox/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT 4.0.0 diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp index bff296de57172..d0924a745716f 100644 --- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp +++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp @@ -1105,6 +1105,7 @@ const ActionsDAG::Node * SerializedPlanParser::parseExpression(ActionsDAG& actio throw Exception(ErrorCodes::BAD_ARGUMENTS, "Doesn't have type or input in cast node."); ActionsDAG::NodeRawConstPtrs args; + String cast_function = "CAST"; const auto & input = rel.cast().input(); args.emplace_back(parseExpression(actions_dag, input)); @@ -1144,10 +1145,15 @@ const ActionsDAG::Node * SerializedPlanParser::parseExpression(ActionsDAG& actio /// Refer to https://github.com/apache/incubator-gluten/issues/4956 args[0] = toFunctionNode(actions_dag, "trim", {args[0]}); } + else if (isString(non_nullable_input_type) && substrait_type.has_bool_()) + { + /// cast(string to boolean) + cast_function = "accurateCastOrNull"; + } /// Common process args.emplace_back(addColumn(actions_dag, std::make_shared(), output_type->getName())); - function_node = toFunctionNode(actions_dag, "CAST", args); + function_node = toFunctionNode(actions_dag, cast_function, args); } actions_dag.addOrReplaceInOutputs(*function_node); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index c98b8e9ec0da4..90f5e895df206 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_08_02 +VELOX_BRANCH=2024_08_06 VELOX_HOME="" OS=`uname -s` diff --git a/gluten-celeborn/clickhouse/pom.xml b/gluten-celeborn/clickhouse/pom.xml index f17f5968d3511..284a8f57282a5 100755 --- a/gluten-celeborn/clickhouse/pom.xml +++ b/gluten-celeborn/clickhouse/pom.xml @@ -5,7 +5,7 @@ gluten-celeborn org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-celeborn/common/pom.xml b/gluten-celeborn/common/pom.xml index 2f274a7bf8e1f..da7e689876594 100755 --- a/gluten-celeborn/common/pom.xml +++ b/gluten-celeborn/common/pom.xml @@ -5,7 +5,7 @@ gluten-celeborn org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-celeborn/package/pom.xml b/gluten-celeborn/package/pom.xml index b982eecd04246..7b18787b4e16d 100644 --- a/gluten-celeborn/package/pom.xml +++ b/gluten-celeborn/package/pom.xml @@ -5,7 +5,7 @@ gluten-celeborn org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/gluten-celeborn/pom.xml b/gluten-celeborn/pom.xml index 480946af0bc88..db82aa81a45a2 100755 --- a/gluten-celeborn/pom.xml +++ b/gluten-celeborn/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-celeborn/velox/pom.xml b/gluten-celeborn/velox/pom.xml index 61498ecafb4ce..99c743e81b080 100755 --- a/gluten-celeborn/velox/pom.xml +++ b/gluten-celeborn/velox/pom.xml @@ -5,7 +5,7 @@ gluten-celeborn org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-core/pom.xml b/gluten-core/pom.xml index 880eddb4ee953..6503f54bce73e 100644 --- a/gluten-core/pom.xml +++ b/gluten-core/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT 4.0.0 diff --git a/gluten-data/pom.xml b/gluten-data/pom.xml index a529eb9b9956e..500708d449694 100644 --- a/gluten-data/pom.xml +++ b/gluten-data/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-delta/pom.xml b/gluten-delta/pom.xml index a66005f2bb2c5..253d5a8e4f4ff 100755 --- a/gluten-delta/pom.xml +++ b/gluten-delta/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-iceberg/pom.xml b/gluten-iceberg/pom.xml index 69630b8cf0eca..d28aac73fa059 100644 --- a/gluten-iceberg/pom.xml +++ b/gluten-iceberg/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ras/common/pom.xml b/gluten-ras/common/pom.xml index 963b350f79b44..6990a855508d7 100644 --- a/gluten-ras/common/pom.xml +++ b/gluten-ras/common/pom.xml @@ -5,7 +5,7 @@ org.apache.gluten gluten-ras - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-ras-common Gluten Ras Common diff --git a/gluten-ras/planner/pom.xml b/gluten-ras/planner/pom.xml index 1ad4af18ea060..652977451a757 100644 --- a/gluten-ras/planner/pom.xml +++ b/gluten-ras/planner/pom.xml @@ -5,7 +5,7 @@ org.apache.gluten gluten-ras - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-ras-planner Gluten Ras Planner diff --git a/gluten-ras/pom.xml b/gluten-ras/pom.xml index 973af760fa87f..570b772235f67 100644 --- a/gluten-ras/pom.xml +++ b/gluten-ras/pom.xml @@ -17,7 +17,7 @@ org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-ras pom diff --git a/gluten-ui/pom.xml b/gluten-ui/pom.xml index d185bb6bf055f..a55d104a0b7a9 100644 --- a/gluten-ui/pom.xml +++ b/gluten-ui/pom.xml @@ -6,7 +6,7 @@ org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-ui diff --git a/gluten-uniffle/package/pom.xml b/gluten-uniffle/package/pom.xml index 64c0fadd3ad6b..e49748e7c8e97 100644 --- a/gluten-uniffle/package/pom.xml +++ b/gluten-uniffle/package/pom.xml @@ -5,7 +5,7 @@ gluten-uniffle org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-uniffle/pom.xml b/gluten-uniffle/pom.xml index ce38bdd2915b3..5ad8c91199496 100644 --- a/gluten-uniffle/pom.xml +++ b/gluten-uniffle/pom.xml @@ -5,7 +5,7 @@ gluten-parent org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-uniffle/velox/pom.xml b/gluten-uniffle/velox/pom.xml index 19865fa6999df..2721188ca76a8 100755 --- a/gluten-uniffle/velox/pom.xml +++ b/gluten-uniffle/velox/pom.xml @@ -5,7 +5,7 @@ gluten-uniffle org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/common/pom.xml b/gluten-ut/common/pom.xml index b40137bafd966..145c66ffa026b 100644 --- a/gluten-ut/common/pom.xml +++ b/gluten-ut/common/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/pom.xml b/gluten-ut/pom.xml index 1e474042d0fa2..90644b832bf82 100644 --- a/gluten-ut/pom.xml +++ b/gluten-ut/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/gluten-ut/spark32/pom.xml b/gluten-ut/spark32/pom.xml index 87aa319740c64..4784ac74fed5b 100644 --- a/gluten-ut/spark32/pom.xml +++ b/gluten-ut/spark32/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/spark33/pom.xml b/gluten-ut/spark33/pom.xml index 5f116ef3bd4cc..079cdf2d0a248 100644 --- a/gluten-ut/spark33/pom.xml +++ b/gluten-ut/spark33/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/spark34/pom.xml b/gluten-ut/spark34/pom.xml index 94d322256f51c..7edd7913ee015 100644 --- a/gluten-ut/spark34/pom.xml +++ b/gluten-ut/spark34/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/spark35/pom.xml b/gluten-ut/spark35/pom.xml index a701bcd819b40..4dcb7de6d0a58 100644 --- a/gluten-ut/spark35/pom.xml +++ b/gluten-ut/spark35/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/gluten-ut/test/pom.xml b/gluten-ut/test/pom.xml index f7d16b8b842c0..52761e988e1e5 100644 --- a/gluten-ut/test/pom.xml +++ b/gluten-ut/test/pom.xml @@ -5,7 +5,7 @@ gluten-ut org.apache.gluten - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml 4.0.0 diff --git a/package/pom.xml b/package/pom.xml index ab87e14805ff5..794118f833ce9 100644 --- a/package/pom.xml +++ b/package/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 6f6b2cd57f673..fd44d06b759c7 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ 4.0.0 org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT pom Gluten Parent Pom diff --git a/shims/common/pom.xml b/shims/common/pom.xml index 959a95fc074fe..86ee768c32938 100644 --- a/shims/common/pom.xml +++ b/shims/common/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten spark-sql-columnar-shims - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/shims/pom.xml b/shims/pom.xml index 61bea7040999e..adc2ca1dad942 100644 --- a/shims/pom.xml +++ b/shims/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten gluten-parent - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/shims/spark32/pom.xml b/shims/spark32/pom.xml index 2a3ed7400c9cb..802b758c8531a 100644 --- a/shims/spark32/pom.xml +++ b/shims/spark32/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten spark-sql-columnar-shims - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/shims/spark33/pom.xml b/shims/spark33/pom.xml index 725d1f8edf01f..f552eddf3fb00 100644 --- a/shims/spark33/pom.xml +++ b/shims/spark33/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten spark-sql-columnar-shims - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/shims/spark34/pom.xml b/shims/spark34/pom.xml index 9cc9bb409beae..5d6a5dd1960f8 100644 --- a/shims/spark34/pom.xml +++ b/shims/spark34/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten spark-sql-columnar-shims - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/shims/spark35/pom.xml b/shims/spark35/pom.xml index 1c79b882bf4ab..66fbb9100ced2 100644 --- a/shims/spark35/pom.xml +++ b/shims/spark35/pom.xml @@ -20,7 +20,7 @@ org.apache.gluten spark-sql-columnar-shims - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT ../pom.xml diff --git a/tools/gluten-it/common/pom.xml b/tools/gluten-it/common/pom.xml index de0d7c2e6c8ca..098b04922b42c 100644 --- a/tools/gluten-it/common/pom.xml +++ b/tools/gluten-it/common/pom.xml @@ -7,11 +7,11 @@ org.apache.gluten gluten-it - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-it-common - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT jar diff --git a/tools/gluten-it/package/pom.xml b/tools/gluten-it/package/pom.xml index d630b774fa30c..87e8db65d10c6 100644 --- a/tools/gluten-it/package/pom.xml +++ b/tools/gluten-it/package/pom.xml @@ -5,7 +5,7 @@ org.apache.gluten gluten-it - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT gluten-it-package Archetype - assembly diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index 5ab633252f305..8ed074af78e90 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -6,7 +6,7 @@ org.apache.gluten gluten-it - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT pom common @@ -23,7 +23,7 @@ 3 0.3.2-incubating 0.9.0 - 1.2.0-SNAPSHOT + 1.3.0-SNAPSHOT 32.0.1-jre 1.1 1.4 diff --git a/tools/workload/tpcds/run_tpcds/run_tpcds.sh b/tools/workload/tpcds/run_tpcds/run_tpcds.sh index 0354f0b90b750..813184307eee5 100755 --- a/tools/workload/tpcds/run_tpcds/run_tpcds.sh +++ b/tools/workload/tpcds/run_tpcds/run_tpcds.sh @@ -37,4 +37,4 @@ cat tpcds_parquet.scala | ${SPARK_HOME}/bin/spark-shell \ # e.g. # --conf spark.gluten.loadLibFromJar=true \ # --jars /PATH_TO_GLUTEN_HOME/package/target/thirdparty-lib/gluten-thirdparty-lib-ubuntu-22.04-x86_64.jar, - # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04_x86_64-1.2.0-SNAPSHOT.jar + # /PATH_TO_GLUTEN_HOME/package/target/gluten-velox-bundle-spark3.3_2.12-ubuntu_22.04_x86_64-1.3.0-SNAPSHOT.jar