Skip to content

Commit

Permalink
[VL] Add validation for coalesce operator
Browse files Browse the repository at this point in the history
  • Loading branch information
zhli1142015 committed Jul 11, 2024
1 parent f3db088 commit a4b816d
Show file tree
Hide file tree
Showing 2 changed files with 22 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -258,6 +258,16 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
checkLengthAndPlan(df, 5)
}

testWithSpecifiedSparkVersion("coalesce validation", Some("3.5")) {
withTempPath {
path =>
val data = "2019-09-09 01:02:03.456789"
val df = Seq(data).toDF("strTs").selectExpr(s"CAST(strTs AS TIMESTAMP_NTZ) AS ts")
df.coalesce(1).write.format("parquet").save(path.getCanonicalPath)
spark.read.parquet(path.getCanonicalPath).collect
}
}

test("groupby") {
val df = runQueryAndCompare(
"select l_orderkey, sum(l_partkey) as sum from lineitem " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,9 @@
*/
package org.apache.gluten.execution

import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.extension.GlutenPlan
import org.apache.gluten.extension.ValidationResult

import org.apache.spark.{Partition, SparkContext, TaskContext}
import org.apache.spark.rdd.RDD
Expand All @@ -32,6 +34,16 @@ case class ColumnarCoalesceExec(numPartitions: Int, child: SparkPlan)

override def supportsColumnar: Boolean = true

override protected def doValidateInternal(): ValidationResult = {
BackendsApiManager.getValidatorApiInstance
.doSchemaValidate(child.schema)
.map {
reason =>
ValidationResult.notOk(s"Found schema check failure for ${child.schema}, due to: $reason")
}
.getOrElse(ValidationResult.ok)
}

override def output: Seq[Attribute] = child.output

override def outputPartitioning: Partitioning = {
Expand Down

0 comments on commit a4b816d

Please sign in to comment.