From 4ea5bc0a4bc53df2d575bea1d63cce87e9db82a5 Mon Sep 17 00:00:00 2001 From: zhaokuo Date: Fri, 19 Jul 2024 16:09:59 +0800 Subject: [PATCH] fix --- .../gluten/backendsapi/clickhouse/CHIteratorApi.scala | 4 ++-- .../apache/gluten/backendsapi/velox/VeloxIteratorApi.scala | 4 ++-- .../scala/org/apache/gluten/backendsapi/IteratorApi.scala | 2 +- .../apache/gluten/execution/BasicScanExecTransformer.scala | 7 ++++++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 01f2ace50240..cd829d04f2cf 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -123,7 +123,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo = { + properties: Map[String, String]): SplitInfo = { partition match { case p: GlutenMergeTreePartition => val partLists = new JArrayList[String]() @@ -185,7 +185,7 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { new JArrayList[JMap[String, String]](), fileFormat, preferredLocations.toList.asJava, - mapAsJavaMap(scan.getProperties) + mapAsJavaMap(properties) ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition: $partition.") diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index b3c34ebf9d51..d8355e1c419f 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -55,7 +55,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo = { + properties: Map[String, String]): SplitInfo = { partition match { case f: FilePartition => val ( @@ -80,7 +80,7 @@ class VeloxIteratorApi extends IteratorApi with Logging { metadataColumns, fileFormat, preferredLocations.toList.asJava, - mapAsJavaMap(scan.getProperties) + mapAsJavaMap(properties) ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition.") diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala index 0ea0904606d8..b78064973123 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala @@ -37,7 +37,7 @@ trait IteratorApi { partitionSchema: StructType, fileFormat: ReadFileFormat, metadataColumnNames: Seq[String], - scan: BasicScanExecTransformer): SplitInfo + properties: Map[String, String]): SplitInfo /** Generate native row partition. */ def genPartitions( diff --git a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala index 3933b0d690b8..04697280d799 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/execution/BasicScanExecTransformer.scala @@ -70,7 +70,12 @@ trait BasicScanExecTransformer extends LeafTransformSupport with BaseDataSource def getSplitInfosFromPartitions(partitions: Seq[InputPartition]): Seq[SplitInfo] = { partitions.map( BackendsApiManager.getIteratorApiInstance - .genSplitInfo(_, getPartitionSchema, fileFormat, getMetadataColumns.map(_.name), this)) + .genSplitInfo( + _, + getPartitionSchema, + fileFormat, + getMetadataColumns.map(_.name), + getProperties)) } override protected def doValidateInternal(): ValidationResult = {