support raw split info

apache · Jul 24, 2024 · 551582d · 551582d
1 parent f069847
commit 551582d
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 3 deletions.
diff --git a/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala b/backends-velox/src/main/scala/org/apache/gluten/backendsapi/velox/VeloxIteratorApi.scala
@@ -57,7 +57,12 @@ class VeloxIteratorApi extends IteratorApi with Logging {
       properties: Map[String, String]): SplitInfo = {
     partition match {
       case f: FilePartition =>
-        new RawSplitInfo(f, partitionSchema, fileFormat, metadataColumnNames.asJava)
+        new RawSplitInfo(
+          f,
+          partitionSchema,
+          fileFormat,
+          metadataColumnNames.asJava,
+          properties.asJava)
       case _ =>
         throw new UnsupportedOperationException(s"Unsupported input partition.")
     }
@@ -107,7 +112,8 @@ class VeloxIteratorApi extends IteratorApi with Logging {
             partitionColumns,
             metadataColumns,
             splitInfo.getFileFormat,
-            new JArrayList[String]()
+            new JArrayList[String](),
+            splitInfo.getProperties
           )
           .toProtobuf
           .toByteArray

diff --git a/gluten-core/src/main/java/org/apache/gluten/substrait/rel/RawSplitInfo.java b/gluten-core/src/main/java/org/apache/gluten/substrait/rel/RawSplitInfo.java
@@ -25,23 +25,27 @@
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 
 public final class RawSplitInfo implements SplitInfo {
 
   private final FilePartition filePartition;
   private final StructType partitionSchema;
   private final LocalFilesNode.ReadFileFormat fileFormat;
   private final List<String> metadataColumn;
+  private final Map<String, String> properties;
 
   public RawSplitInfo(
       FilePartition filePartition,
       StructType partitionSchema,
       LocalFilesNode.ReadFileFormat fileFormat,
-      List<String> metadataColumn) {
+      List<String> metadataColumn,
+      Map<String, String> properties) {
     this.filePartition = filePartition;
     this.partitionSchema = partitionSchema;
     this.fileFormat = fileFormat;
     this.metadataColumn = metadataColumn;
+    this.properties = properties;
   }
 
   public FilePartition getFilePartition() {
@@ -60,6 +64,10 @@ public List<String> getMetadataColumn() {
     return metadataColumn;
   }
 
+  public Map<String, String> getProperties() {
+    return properties;
+  }
+
   @Override
   public List<String> preferredLocations() {
     return Arrays.asList(SoftAffinity.getFilePartitionLocations(filePartition));