From c412e0a1587fb0527e5072343c891cd13a44c742 Mon Sep 17 00:00:00 2001 From: zhli1142015 Date: Thu, 11 Jul 2024 18:55:54 +0800 Subject: [PATCH] adress comments --- .../execution/RowToVeloxColumnarExec.scala | 9 --------- .../apache/gluten/extension/GlutenPlan.scala | 18 +++++++++++------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala index 29478fe9dbd7..2f3e88f9af9c 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/execution/RowToVeloxColumnarExec.scala @@ -17,9 +17,7 @@ package org.apache.gluten.execution import org.apache.gluten.GlutenConfig -import org.apache.gluten.backendsapi.BackendsApiManager import org.apache.gluten.columnarbatch.ColumnarBatches -import org.apache.gluten.exception.GlutenException import org.apache.gluten.exec.Runtimes import org.apache.gluten.memory.arrow.alloc.ArrowBufferAllocators import org.apache.gluten.utils.ArrowAbiUtil @@ -47,13 +45,6 @@ import scala.collection.mutable.ListBuffer case class RowToVeloxColumnarExec(child: SparkPlan) extends RowToColumnarExecBase(child = child) { override def doExecuteColumnarInternal(): RDD[ColumnarBatch] = { - BackendsApiManager.getValidatorApiInstance.doSchemaValidate(schema).foreach { - reason => - throw new GlutenException( - s"Input schema contains unsupported type when convert row to columnar for $schema " + - s"due to $reason") - } - val numInputRows = longMetric("numInputRows") val numOutputBatches = longMetric("numOutputBatches") val convertTime = longMetric("convertTime") diff --git a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala index d29b29b0c104..71a76ff63dd1 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/extension/GlutenPlan.scala @@ -63,15 +63,19 @@ trait GlutenPlan extends SparkPlan with Convention.KnownBatchType with LogLevelU * Validate whether this SparkPlan supports to be transformed into substrait node in Native Code. */ final def doValidate(): ValidationResult = { + val schemaVaidationResult = BackendsApiManager.getValidatorApiInstance + .doSchemaValidate(schema) + .map { + reason => ValidationResult.notOk(s"Found schema check failure for $schema, due to: $reason") + } + .getOrElse(ValidationResult.ok) + if (!schemaVaidationResult.isValid) { + TestStats.addFallBackClassName(this.getClass.toString) + return schemaVaidationResult + } try { TransformerState.enterValidation - val res = BackendsApiManager.getValidatorApiInstance - .doSchemaValidate(schema) - .map { - reason => - ValidationResult.notOk(s"Found schema check failure for $schema, due to: $reason") - } - .getOrElse(doValidateInternal()) + val res = doValidateInternal() if (!res.isValid) { TestStats.addFallBackClassName(this.getClass.toString) }