Support decimal allow precision loss

apache · Aug 30, 2024 · d03a96f · d03a96f
1 parent 531e2d8
commit d03a96f
Show file tree

Hide file tree

Showing 4 changed files with 20 additions and 6 deletions.
diff --git a/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala b/...ends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxSparkPlanExecApi.scala
@@ -159,7 +159,15 @@ class VeloxSparkPlanExecApi extends SparkPlanExecApi {
     } else if (SparkShimLoader.getSparkShims.withAnsiEvalMode(original)) {
       throw new GlutenNotSupportException(s"$substraitExprName with ansi mode is not supported")
     } else {
-      GenericExpressionTransformer(substraitExprName, Seq(left, right), original)
+      if (
+        left.dataType.isInstanceOf[DecimalType] && right.dataType
+          .isInstanceOf[DecimalType] && !SQLConf.get.decimalOperationsAllowPrecisionLoss
+      ) {
+        val newName = "not_allow_precision_loss_"
+        GenericExpressionTransformer(newName, Seq(left, right), original)
+      } else {
+        GenericExpressionTransformer(substraitExprName, Seq(left, right), original)
+      }
     }
   }
 

diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -445,10 +445,6 @@ std::unordered_map<std::string, std::string> WholeStageResultIterator::getQueryC
     // Adjust timestamp according to the above configured session timezone.
     configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = "true";
 
-    // To align with Spark's behavior, allow decimal precision loss or not.
-    configs[velox::core::QueryConfig::kSparkDecimalOperationsAllowPrecisionLoss] =
-        veloxCfg_->get<std::string>(kAllowPrecisionLoss, "true");
-
     {
       // partial aggregation memory config
       auto offHeapMemory = veloxCfg_->get<int64_t>(kSparkTaskOffHeapMemory, facebook::velox::memory::kMaxMemory);

diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc b/cpp/velox/operators/functions/RegistrationAllFunctions.cc
@@ -26,6 +26,7 @@
 #include "velox/functions/prestosql/aggregates/RegisterAggregateFunctions.h"
 #include "velox/functions/prestosql/registration/RegistrationFunctions.h"
 #include "velox/functions/prestosql/window/WindowFunctionsRegistration.h"
+#include "velox/functions/sparksql/DecimalArithmetic.h"
 #include "velox/functions/sparksql/Hash.h"
 #include "velox/functions/sparksql/Rand.h"
 #include "velox/functions/sparksql/Register.h"
@@ -74,6 +75,14 @@ void registerFunctionOverwrite() {
 
   velox::functions::registerPrestoVectorFunctions();
 }
+
+void registerFunctionForConfig() {
+  const std::string prefix = "not_allow_precision_loss_";
+  velox::functions::sparksql::registerDecimalAdd(prefix, false);
+  velox::functions::sparksql::registerDecimalSubtract(prefix, false);
+  velox::functions::sparksql::registerDecimalMultiply(prefix, false);
+  velox::functions::sparksql::registerDecimalDivide(prefix, false);
+}
 } // namespace
 
 void registerAllFunctions() {
@@ -87,6 +96,7 @@ void registerAllFunctions() {
   // Using function overwrite to handle function names mismatch between Spark
   // and Velox.
   registerFunctionOverwrite();
+  registerFunctionForConfig();
 }
 
 } // namespace gluten
diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh
@@ -17,7 +17,7 @@
 set -exu
 
 VELOX_REPO=https://github.com/oap-project/velox.git
-VELOX_BRANCH=2024_08_28
+VELOX_BRANCH=2024_08_28_fix
 VELOX_HOME=""
 
 OS=`uname -s`