From c650efde3926b287563d9502ff6c6c2a1e4aee8c Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Fri, 19 Jul 2024 16:09:59 +0800 Subject: [PATCH] fix --- .../gluten/backendsapi/clickhouse/CHIteratorApi.scala | 4 ++-- .../apache/gluten/backendsapi/velox/VeloxIteratorApi.scala | 4 ++-- .../scala/org/apache/gluten/backendsapi/IteratorApi.scala | 2 +- .../apache/gluten/execution/BasicScanExecTransformer.scala | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 5f2bc19c377d1..ac9c633bcdd27 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -123,7 +123,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo = { + properties: Map[String, String]): SplitInfo = { partition match { case p: GlutenMergeTreePartition => val partLists = new JArrayList[String]() @@ -185,7 +185,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { new JArrayList[JMap[String, String]](), fileFormat, preferredLocations.toList.asJava, - mapAsJavaMap(scan.getProperties) + mapAsJavaMap(properties) ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition: $partition.") diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index c34abf32fe7d9..cc3cabb7c5e86 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -55,7 +55,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo = { + properties: Map[String, String]): SplitInfo = { partition match { case f: FilePartition => val ( @@ -80,7 +80,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { metadataColumns, fileFormat, preferredLocations.toList.asJava, - mapAsJavaMap(scan.getProperties) + mapAsJavaMap(properties) ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition.") diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala index 4f7fe20ad51bd..ee89d3d4616f4 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala @@ -37,7 +37,7 @@ trait IteratorApi { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo + properties: Map[String, String]): SplitInfo /** Generate native row partition. */ def genPartitions( diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 3933b0d690b84..04697280d7996 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -70,7 +70,12 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource def getSplitInfosFromPartitions(partitions: Seq[InputPartition]): Seq[SplitInfo] = { partitions.map( BackendsApiManager.getIteratorApiInstance - .genSplitInfo(_, getPartitionSchema, fileFormat, getMetadataColumns.map(_.name), this)) + .genSplitInfo( + _, + getPartitionSchema, + fileFormat, + getMetadataColumns.map(_.name), + getProperties)) } override protected def doValidateInternal(): ValidationResult = {