diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala index 4b9ec739028f6..5f2bc19c377d1 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHIteratorApi.scala @@ -122,7 +122,8 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { partition: InputPartition, partitionSchema: StructType, fileFormat: ReadFileFormat, - metadataColumnNames: Seq[String]): SplitInfo = { + metadataColumnNames: Seq[String], + scan: BasicScanExecTransformer): SplitInfo = { partition match { case p: GlutenMergeTreePartition => val partLists = new JArrayList[String]() @@ -183,7 +184,8 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { partitionColumns, new JArrayList[JMap[String, String]](), fileFormat, - preferredLocations.toList.asJava + preferredLocations.toList.asJava, + mapAsJavaMap(scan.getProperties) ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition: $partition.") @@ -209,7 +211,6 @@ class CHIteratorApi extends IteratorApi with Logging with LogLevelUtil { split match { case filesNode: LocalFilesNode => setFileSchemaForLocalFiles(filesNode, scans(i)) - filesNode.setFileReadProperties(mapAsJavaMap(scans(i).getProperties)) filesNode.getPaths.forEach(f => files += f) filesNode.toProtobuf.toByteArray case extensionTableNode: ExtensionTableNode => diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala index 613e539456ecf..c34abf32fe7d9 100644 --- a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala +++ b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala @@ -54,7 +54,8 @@ class VeloxIteratorApi extends IteratorApi with Logging { partition: InputPartition, partitionSchema: StructType, fileFormat: ReadFileFormat, - metadataColumnNames: Seq[String]): SplitInfo = { + metadataColumnNames: Seq[String], + scan: BasicScanExecTransformer): SplitInfo = { partition match { case f: FilePartition => val ( @@ -78,7 +79,9 @@ class VeloxIteratorApi extends IteratorApi with Logging { partitionColumns, metadataColumns, fileFormat, - preferredLocations.toList.asJava) + preferredLocations.toList.asJava, + mapAsJavaMap(scan.getProperties) + ) case _ => throw new UnsupportedOperationException(s"Unsupported input partition.") } diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java index 7e085f81f4e66..a58f5e043503a 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesBuilder.java @@ -32,7 +32,8 @@ public static LocalFilesNode makeLocalFiles( List> partitionColumns, List> metadataColumns, LocalFilesNode.ReadFileFormat fileFormat, - List preferredLocations) { + List preferredLocations, + Map properties) { return new LocalFilesNode( index, paths, @@ -43,7 +44,8 @@ public static LocalFilesNode makeLocalFiles( partitionColumns, metadataColumns, fileFormat, - preferredLocations); + preferredLocations, + properties); } public static LocalFilesNode makeLocalFiles(String iterPath) { diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java index fa9f3d51612bb..03c31ccca930f 100644 --- a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java +++ b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/LocalFilesNode.java @@ -67,7 +67,9 @@ public enum ReadFileFormat { List> partitionColumns, List> metadataColumns, ReadFileFormat fileFormat, - List preferredLocations) { + List preferredLocations, + Map properties + ) { this.index = index; this.paths.addAll(paths); this.starts.addAll(starts); @@ -78,6 +80,7 @@ public enum ReadFileFormat { this.partitionColumns.addAll(partitionColumns); this.metadataColumns.addAll(metadataColumns); this.preferredLocations.addAll(preferredLocations); + this.fileReadProperties= properties; } LocalFilesNode(String iterPath) { diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala index 53dc8f47861fb..4f7fe20ad51bd 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/IteratorApi.scala @@ -36,7 +36,8 @@ trait IteratorApi { partition: InputPartition, partitionSchema: StructType, fileFormat: ReadFileFormat, - metadataColumnNames: Seq[String]): SplitInfo + metadataColumnNames: Seq[String], + scan: BasicScanExecTransformer): SplitInfo /** Generate native row partition. */ def genPartitions(