From 1efa6671fc129e8f4853f99d27a647504e25036d Mon Sep 17 00:00:00 2001 From: PHILO-HE Date: Fri, 24 Nov 2023 22:57:20 +0800 Subject: [PATCH] Separate the impl for backends --- .../backendsapi/clickhouse/CHSparkPlanExecApi.scala | 9 ++++++++- .../execution/GlutenClickHouseTPCHParquetSuite.scala | 4 ++-- .../io/glutenproject/backendsapi/SparkPlanExecApi.scala | 7 +++++++ .../glutenproject/expression/ExpressionConverter.scala | 2 +- .../expression/UnaryExpressionTransformer.scala | 4 ---- 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala index 60c19a469ddf..450429b67429 100644 --- a/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/io/glutenproject/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -188,7 +188,14 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { left: ExpressionTransformer, right: ExpressionTransformer, original: GetMapValue): ExpressionTransformer = - new GetMapValueTransformer(substraitExprName, left, right, original.failOnError, original) + GetMapValueTransformer(substraitExprName, left, right, original.failOnError, original) + + override def genRandTransformer( + substraitExprName: String, + explicitSeed: ExpressionTransformer, + original: Rand): ExpressionTransformer = { + GenericExpressionTransformer(substraitExprName, Seq(explicitSeed), original) + } /** * Generate ShuffleDependency for ColumnarShuffleExchangeExec. diff --git a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetSuite.scala b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetSuite.scala index 2f65a7687e20..454f29c8bab4 100644 --- a/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetSuite.scala +++ b/backends-clickhouse/src/test/scala/io/glutenproject/execution/GlutenClickHouseTPCHParquetSuite.scala @@ -376,8 +376,8 @@ class GlutenClickHouseTPCHParquetSuite extends GlutenClickHouseTPCHAbstractSuite } test("test 'function rand'") { - // TODO: add rand(1), rand(null) test back. - runSql("select rand() from range(10)")(checkOperatorMatch[ProjectExecTransformer]) + runSql("select rand(), rand(1), rand(null) from range(10)")( + checkOperatorMatch[ProjectExecTransformer]) } test("test 'function date_add/date_sub/datediff'") { diff --git a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala index f910e16d9a0b..4d825906c8c4 100644 --- a/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/io/glutenproject/backendsapi/SparkPlanExecApi.scala @@ -127,6 +127,13 @@ trait SparkPlanExecApi { GenericExpressionTransformer(substraitExprName, Seq(srcExpr, regexExpr, limitExpr), original) } + def genRandTransformer( + substraitExprName: String, + explicitSeed: ExpressionTransformer, + original: Rand): ExpressionTransformer = { + RandTransformer(substraitExprName, explicitSeed, original) + } + /** Generate an expression transformer to transform GetMapValue to Substrait. */ def genGetMapValueTransformer( substraitExprName: String, diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala index b24baa5ff96f..7fbc34e973bc 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/ExpressionConverter.scala @@ -391,7 +391,7 @@ object ExpressionConverter extends SQLConfHelper with Logging { replaceWithExpressionTransformer(m.child, attributeSeq), m) case rand: Rand => - RandTransformer( + BackendsApiManager.getSparkPlanExecApiInstance.genRandTransformer( substraitExprName, replaceWithExpressionTransformer(rand.child, attributeSeq), rand) diff --git a/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala b/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala index 8e6b0e56849b..6059e7e857f3 100644 --- a/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala +++ b/gluten-core/src/main/scala/io/glutenproject/expression/UnaryExpressionTransformer.scala @@ -220,10 +220,6 @@ case class RandTransformer( functionMap, ConverterUtils.makeFuncName(substraitExprName, Seq(original.child.dataType))) val inputNodes = Lists.newArrayList[ExpressionNode]() -// inputNodes.add(explicitSeed.doTransform(args)) -// val partitionIndex = TaskContext.getPartitionId() -// val partitionIndexNode = ExpressionBuilder.makeLiteral(partitionIndex, IntegerType, false) -// inputNodes.add(partitionIndexNode) val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) ExpressionBuilder.makeScalarFunction(functionId, inputNodes, typeNode) }