Skip to content

Commit

Permalink
adress comments
Browse files Browse the repository at this point in the history
  • Loading branch information
zhli1142015 committed Jul 11, 2024
1 parent 6937570 commit c412e0a
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 16 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
package org.apache.gluten.execution

import org.apache.gluten.GlutenConfig
import org.apache.gluten.backendsapi.BackendsApiManager
import org.apache.gluten.columnarbatch.ColumnarBatches
import org.apache.gluten.exception.GlutenException
import org.apache.gluten.exec.Runtimes
import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators
import org.apache.gluten.utils.ArrowAbiUtil
Expand Down Expand Up @@ -47,13 +45,6 @@ import scala.collection.mutable.ListBuffer
case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBase(child = child) {

override def doExecuteColumnarInternal(): RDD[ColumnarBatch] = {
BackendsApiManager.getValidatorApiInstance.doSchemaValidate(schema).foreach {
reason =>
throw new GlutenException(
s"Input schema contains unsupported type when convert row to columnar for $schema " +
s"due to $reason")
}

val numInputRows = longMetric("numInputRows")
val numOutputBatches = longMetric("numOutputBatches")
val convertTime = longMetric("convertTime")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,15 +63,19 @@ trait GlutenPlan extends SparkPlan with Convention.KnownBatchType with LogLevelU
* Validate whether this SparkPlan supports to be transformed into substrait node in Native Code.
*/
final def doValidate(): ValidationResult = {
val schemaVaidationResult = BackendsApiManager.getValidatorApiInstance
.doSchemaValidate(schema)
.map {
reason => ValidationResult.notOk(s"Found schema check failure for $schema, due to: $reason")
}
.getOrElse(ValidationResult.ok)
if (!schemaVaidationResult.isValid) {
TestStats.addFallBackClassName(this.getClass.toString)
return schemaVaidationResult
}
try {
TransformerState.enterValidation
val res = BackendsApiManager.getValidatorApiInstance
.doSchemaValidate(schema)
.map {
reason =>
ValidationResult.notOk(s"Found schema check failure for $schema, due to: $reason")
}
.getOrElse(doValidateInternal())
val res = doValidateInternal()
if (!res.isValid) {
TestStats.addFallBackClassName(this.getClass.toString)
}
Expand Down

0 comments on commit c412e0a

Please sign in to comment.