diff --git a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/gluten/KapExpressionTransformer.scala b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/gluten/KapExpressionTransformer.scala index 0ccbad53ce5..1f8614eba94 100644 --- a/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/gluten/KapExpressionTransformer.scala +++ b/src/spark-project/sparder/src/main/scala/org/apache/spark/sql/catalyst/expressions/gluten/KapExpressionTransformer.scala @@ -17,7 +17,6 @@ package org.apache.spark.sql.catalyst.expressions.gluten import com.google.common.collect.Lists -import org.apache.gluten.backendsapi.clickhouse.CHBackendSettings import org.apache.gluten.exception.GlutenNotSupportException import org.apache.gluten.expression._ import org.apache.gluten.extension.ExpressionExtensionTrait @@ -25,89 +24,11 @@ import org.apache.gluten.substrait.expression.{ExpressionBuilder, ExpressionNode import org.apache.spark.internal.Logging import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.aggregate._ -import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types.{BinaryType, DataType, LongType, StringType} +import org.apache.spark.sql.types.{BinaryType, DataType, LongType} import org.apache.spark.sql.udaf._ -import org.apache.spark.unsafe.types.UTF8String -import java.util.Locale import scala.collection.mutable.ListBuffer -case class FloorDateTimeExpressionTransformer( - substraitExprName: String, - format: ExpressionTransformer, - timestamp: ExpressionTransformer, - timeZoneId: Option[String] = None, - original: FloorDateTime) extends ExpressionTransformerWithOrigin with Logging { - - override def doTransform(args: java.lang.Object): ExpressionNode = { - // The format must be constant string in the fucntion date_trunc of ch. - if (!original.format.foldable) { - throw new UnsupportedOperationException( - s"The format ${original.format} must be constant string.") - } - - val formatStr = original.format.eval().asInstanceOf[UTF8String] - if (formatStr == null) { - throw new UnsupportedOperationException("The format is null.") - } - - val (newFormatStr, timeZoneIgnore) = formatStr.toString.toLowerCase(Locale.ROOT) match { - case "second" => ("second", false) - case "minute" => ("minute", false) - case "hour" => ("hour", false) - case "day" | "dd" => ("day", false) - case "week" => ("week", true) - case "mon" | "month" | "mm" => ("month", true) - case "quarter" => ("quarter", true) - case "year" | "yyyy" | "yy" => ("year", true) - // Can not support now. - // case "microsecond" => "microsecond" - // case "millisecond" => "millisecond" - case _ => throw new UnsupportedOperationException(s"The format $formatStr is invalidate.") - } - - // Currently, data_trunc function can not support to set the specified timezone, - // which is different with session_time_zone. - if ( - timeZoneIgnore && timeZoneId.nonEmpty && - !timeZoneId.get.equalsIgnoreCase( - SQLConf.get.getConfString( - s"${CHBackendSettings.getBackendConfigPrefix}.runtime_config.timezone") - ) - ) { - throw new UnsupportedOperationException( - s"It doesn't support trunc the format $newFormatStr with the specified timezone " + - s"${timeZoneId.get}.") - } - - val timestampNode = timestamp.doTransform(args) - val lowerFormatNode = ExpressionBuilder.makeStringLiteral(newFormatStr) - - val functionMap = args.asInstanceOf[java.util.HashMap[String, java.lang.Long]] - - val dataTypes = if (timeZoneId.nonEmpty) { - Seq(original.format.dataType, original.timestamp.dataType, StringType) - } else { - Seq(original.format.dataType, original.timestamp.dataType) - } - - val functionId = ExpressionBuilder.newScalarFunction( - functionMap, - ConverterUtils.makeFuncName(substraitExprName, dataTypes)) - - val expressionNodes = new java.util.ArrayList[ExpressionNode]() - expressionNodes.add(lowerFormatNode) - expressionNodes.add(timestampNode) - if (timeZoneId != None) { - expressionNodes.add(ExpressionBuilder.makeStringLiteral(timeZoneId.get)) - } - - val typeNode = ConverterUtils.getTypeNode(original.dataType, original.nullable) - ExpressionBuilder.makeScalarFunction(functionId, expressionNodes, typeNode) - } -} - case class KeBitmapFunctionTransformer( substraitExprName: String, child: ExpressionTransformer, @@ -148,14 +69,6 @@ case class CustomerExpressionTransformer() extends ExpressionExtensionTrait { substraitExprName: String, expr: Expression, attributeSeq: Seq[Attribute]): ExpressionTransformer = expr match { - case floorDateTime: FloorDateTime => - FloorDateTimeExpressionTransformer( - substraitExprName, - ExpressionConverter.replaceWithExpressionTransformer(floorDateTime.format, attributeSeq), - ExpressionConverter.replaceWithExpressionTransformer(floorDateTime.timestamp, attributeSeq), - floorDateTime.timeZoneId, - floorDateTime - ) case preciseCardinality: PreciseCardinality => KeBitmapFunctionTransformer( substraitExprName, @@ -171,7 +84,7 @@ case class CustomerExpressionTransformer() extends ExpressionExtensionTrait { preciseCountDistinctDecode ) case kylinTimestampAdd: KylinTimestampAdd => - new TimestampAddTransformer( + TimestampAddTransformer( ExpressionConverter .replaceWithExpressionTransformer(kylinTimestampAdd.left, attributeSeq), ExpressionConverter @@ -180,7 +93,7 @@ case class CustomerExpressionTransformer() extends ExpressionExtensionTrait { .replaceWithExpressionTransformer(kylinTimestampAdd.right, attributeSeq), kylinTimestampAdd ) - case other => + case _ => throw new UnsupportedOperationException( s"${expr.getClass} or $expr is not currently supported.") }