From 913d7800120958abdc9ff5be0afaa19c13a0add4 Mon Sep 17 00:00:00 2001 From: Yangyang Gao Date: Wed, 6 Dec 2023 15:58:17 +0800 Subject: [PATCH] fix issue after merge main --- .../io/glutenproject/execution/BasicScanExecTransformer.scala | 4 +++- .../io/glutenproject/execution/IcebergScanTransformer.scala | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala index bd7185af0e4ca..f0f9b1f7b8ef1 100644 --- a/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/io/glutenproject/execution/BasicScanExecTransformer.scala @@ -26,7 +26,7 @@ import io.glutenproject.substrait.rel.{ReadRelNode, RelBuilder, SplitInfo} import io.glutenproject.substrait.rel.LocalFilesNode.ReadFileFormat import org.apache.spark.rdd.RDD -import org.apache.spark.sql.catalyst.expressions.{And, Attribute, Expression} +import org.apache.spark.sql.catalyst.expressions.{And, Attribute, AttributeReference, Expression} import org.apache.spark.sql.vectorized.ColumnarBatch import com.google.common.collect.Lists @@ -34,6 +34,7 @@ import com.google.common.collect.Lists import scala.collection.JavaConverters._ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource { + import org.apache.spark.sql.catalyst.util._ /** Returns the filters that can be pushed down to native file scan */ def filterExprs(hasMetadataColFilters: Boolean = true): Seq[Expression] @@ -60,6 +61,7 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource getPartitions.map( BackendsApiManager.getIteratorApiInstance .genSplitInfo(_, getPartitionSchema, fileFormat, getMetadataColumns.map(_.name))) + } def doExecuteColumnarInternal(): RDD[ColumnarBatch] = { val numOutputRows = longMetric("outputRows") diff --git a/gluten-iceberg/src/main/scala/io/glutenproject/execution/IcebergScanTransformer.scala b/gluten-iceberg/src/main/scala/io/glutenproject/execution/IcebergScanTransformer.scala index ee4835ddc20e4..d3179bf30e48e 100644 --- a/gluten-iceberg/src/main/scala/io/glutenproject/execution/IcebergScanTransformer.scala +++ b/gluten-iceberg/src/main/scala/io/glutenproject/execution/IcebergScanTransformer.scala @@ -41,7 +41,7 @@ class IcebergScanTransformer( runtimeFilters = runtimeFilters, table = table) { - override def filterExprs(): Seq[Expression] = Seq.empty + override def filterExprs(hasMetataColFilters: Boolean): Seq[Expression] = Seq.empty override def getPartitionSchema: StructType = new StructType()