From bb88e9f058c8b0c421a6aa68055358aea255b0a6 Mon Sep 17 00:00:00 2001 From: Yuan Zhou Date: Fri, 25 Aug 2023 09:44:49 +0800 Subject: [PATCH] support allowDecimalPrecisionLoss Signed-off-by: Yuan Zhou --- cpp/velox/compute/WholeStageResultIterator.cc | 2 ++ ep/build-velox/src/get_velox.sh | 5 +++++ .../io/glutenproject/expression/ExpressionConverter.scala | 8 -------- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc index 214d3cab94a57..57ce61f1881c1 100644 --- a/cpp/velox/compute/WholeStageResultIterator.cc +++ b/cpp/velox/compute/WholeStageResultIterator.cc @@ -444,6 +444,8 @@ std::unordered_map WholeStageResultIterator::getQueryC veloxCfg_->get(kSessionTimezone, defaultTimezone); // Adjust timestamp according to the above configured session timezone. configs[velox::core::QueryConfig::kAdjustTimestampToTimezone] = std::to_string(true); + // To align with Spark's behavior, allow decimal precision loss or not. + configs[velox::core::QueryConfig::kAllowPrecisionLoss] = getConfigValue(confMap_, kAllowPrecisionLoss, "true"); // Align Velox size function with Spark. configs[velox::core::QueryConfig::kSparkLegacySizeOfNull] = std::to_string(veloxCfg_->get(kLegacySize, true)); diff --git a/ep/build-velox/src/get_velox.sh b/ep/build-velox/src/get_velox.sh index 489481581ff94..7996feffbcad3 100755 --- a/ep/build-velox/src/get_velox.sh +++ b/ep/build-velox/src/get_velox.sh @@ -16,8 +16,13 @@ set -exu +<<<<<<< HEAD VELOX_REPO=https://github.com/oap-project/velox.git VELOX_BRANCH=2024_02_27 +======= +VELOX_REPO=https://github.com/zhouyuan/velox.git +VELOX_BRANCH=wip_decimal_precision_loss +>>>>>>> 43f847d76... support allowDecimalPrecisionLoss VELOX_HOME="" #Set on run gluten on HDFS diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala index f75d7fe74cdb2..ca4877bd29894 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala @@ -471,14 +471,6 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformerInternal(_, attributeSeq, expressionsMap)), expr) case b: BinaryArithmetic if DecimalArithmeticUtil.isDecimalArithmetic(b) => - // PrecisionLoss=true: velox support / ch not support - // PrecisionLoss=false: velox not support / ch support - // TODO ch support PrecisionLoss=true - if (!BackendsApiManager.getSettings.allowDecimalArithmetic) { - throw new UnsupportedOperationException( - s"Not support ${SQLConf.DECIMAL_OPERATIONS_ALLOW_PREC_LOSS.key} " + - s"${conf.decimalOperationsAllowPrecisionLoss} mode") - } val rescaleBinary = if (BackendsApiManager.getSettings.rescaleDecimalLiteral) { DecimalArithmeticUtil.rescaleLiteral(b) } else {