Use two-dimensional vector to store delete files

apache · Mar 13, 2024 · d2ebfb7 · d2ebfb7
1 parent 032bf3e
commit d2ebfb7
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 7 deletions.
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -156,9 +156,7 @@ WholeStageResultIterator::WholeStageResultIterator(
       if (auto icebergSplitInfo = std::dynamic_pointer_cast<IcebergSplitInfo>(scanInfo)) {
         // Set Iceberg split.
         std::unordered_map<std::string, std::string> customSplitInfo{{"table_format", "hive-iceberg"}};
-        auto deleteFilesFind = icebergSplitInfo->deleteFilesMap.find(paths[idx]);
-        auto deleteFiles = deleteFilesFind != icebergSplitInfo->deleteFilesMap.end() ? deleteFilesFind->second
-                                                                                     : std::vector<IcebergDeleteFile>{};
+        auto deleteFiles = icebergSplitInfo->deleteFilesVec[idx];
         split = std::make_shared<velox::connector::hive::iceberg::HiveIcebergSplit>(
             kHiveConnectorId,
             paths[idx],

diff --git a/cpp/velox/compute/iceberg/IcebergPlanConverter.cc b/cpp/velox/compute/iceberg/IcebergPlanConverter.cc
@@ -72,7 +72,10 @@ std::shared_ptr<IcebergSplitInfo> IcebergPlanConverter::parseIcebergSplitInfo(
       deletes.emplace_back(IcebergDeleteFile(
           fileContent, deleteFile.filepath(), format, deleteFile.recordcount(), deleteFile.filesize()));
     }
-    icebergSplitInfo->deleteFilesMap[file.uri_file()] = std::move(deletes);
+    icebergSplitInfo->deleteFilesVec.emplace_back(deletes);
+  } else {
+    // Add an empty delete files vector to indicate that this data file has no delete file.
+    icebergSplitInfo->deleteFilesVec.emplace_back(std::vector<IcebergDeleteFile>{});
   }
 
   return icebergSplitInfo;

diff --git a/cpp/velox/compute/iceberg/IcebergPlanConverter.h b/cpp/velox/compute/iceberg/IcebergPlanConverter.h
@@ -24,9 +24,12 @@ using namespace facebook::velox::connector::hive::iceberg;
 
 namespace gluten {
 struct IcebergSplitInfo : SplitInfo {
-  std::unordered_map<std::string, std::vector<IcebergDeleteFile>> deleteFilesMap;
+  std::vector<std::vector<IcebergDeleteFile>> deleteFilesVec;
 
-  IcebergSplitInfo(const SplitInfo& splitInfo) : SplitInfo(splitInfo) {}
+  IcebergSplitInfo(const SplitInfo& splitInfo) : SplitInfo(splitInfo) {
+    // Reserve the actual size of the deleteFilesVec.
+    deleteFilesVec.reserve(splitInfo.paths.capacity());
+  }
 };
 
 class IcebergPlanConverter {

diff --git a/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxIcebergSuite.scala b/gluten-iceberg/src/test/scala/io/glutenproject/execution/VeloxIcebergSuite.scala
@@ -200,7 +200,7 @@ class VeloxIcebergSuite extends WholeStageTransformerSuite {
             |update iceberg_mor_tb set name = 'new_a2' where id = 'a2';
             |""".stripMargin
         )
-        
+
         // Merge into.
         spark.sql(
           """