From d03a96fd2a51639596dcef4f25579851c9ca2278 Mon Sep 17 00:00:00 2001 From: Chengcheng Jin Date: Fri, 30 Aug 2024 10:49:08 +0000 Subject: [PATCH] Support decimal allow precision loss --- .../backendsapi/velox/VeloxSparkPlanExecApi.scala | 10 +++++++++- cpp/velox/compute/WholeStageResultIterator.cc | 4 ---- .../operators/functions/RegistrationAllFunctions.cc | 10 ++++++++++ ep/build-velox/src/get_velox.sh | 2 +- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala index 554b3791dad3..4755adc91245 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala @@ -159,7 +159,15 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi { } else if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) { throw new GlutenNotSupportException(s"$substraitExprName with ansi mode is not supported") } else { - GenericExpressionTransformer(substraitExprName, Seq(left, right), original) + if ( + left.dataType.isInstanceOf[DecimalType] && right.dataType + .isInstanceOf[DecimalType] && !SQLConf.get.decimalOperationsAllowPrecisionLoss + ) { + val newName = "not_allow_precision_loss_" + GenericExpressionTransformer(newName, Seq(left, right), original) + } else { + GenericExpressionTransformer(substraitExprName, Seq(left, right), original) + } } } diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index c306564dc8d9..1a990ac75e0a 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -445,10 +445,6 @@ std::unordered_map WholeStageResultIterator::getQueryC // Adjust timestamp according to the above configured session timezone. configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true"; - // To align with Spark's behavior, allow decimal precision loss or not. - configs[velox::core::QueryConfig::kSparkDecimalOperationsAllowPrecisionLoss] = - veloxCfg_->get(kAllowPrecisionLoss, "true"); - { // partial aggregation memory config auto offHeapMemory = veloxCfg_->get(kSparkTaskOffHeapMemory, facebook::velox::memory::kMaxMemory); diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc index 6b6564fa4aa3..6e2f90f0105b 100644 --- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc +++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc @@ -26,6 +26,7 @@ #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h" #include "velox/functions/prestosql/registration/RegistrationFunctions.h" #include "velox/functions/prestosql/window/WindowFunctionsRegistration.h" +#include "velox/functions/sparksql/DecimalArithmetic.h" #include "velox/functions/sparksql/Hash.h" #include "velox/functions/sparksql/Rand.h" #include "velox/functions/sparksql/Register.h" @@ -74,6 +75,14 @@ void registerFunctionOverwrite() { velox::functions::registerPrestoVectorFunctions(); } + +void registerFunctionForConfig() { + const std::string prefix = "not_allow_precision_loss_"; + velox::functions::sparksql::registerDecimalAdd(prefix, false); + velox::functions::sparksql::registerDecimalSubtract(prefix, false); + velox::functions::sparksql::registerDecimalMultiply(prefix, false); + velox::functions::sparksql::registerDecimalDivide(prefix, false); +} } // namespace void registerAllFunctions() { @@ -87,6 +96,7 @@ void registerAllFunctions() { // Using function overwrite to handle function names mismatch between Spark // and Velox. registerFunctionOverwrite(); + registerFunctionForConfig(); } } // namespace gluten diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 8113e42169dc..12186a56bfa8 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -17,7 +17,7 @@ set -exu VELOX_REPO=https://github.com/oap-project/velox.git -VELOX_BRANCH=2024_08_28 +VELOX_BRANCH=2024_08_28_fix VELOX_HOME="" OS=`uname -s`