Run benchmark

apache · Sep 20, 2024 · d0f28a6 · d0f28a6
1 parent 326af0a
commit d0f28a6
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 12 deletions.
diff --git a/core/src/jmh/java/org/apache/iceberg/ReplaceDeleteFilesBenchmark.java b/core/src/jmh/java/org/apache/iceberg/ReplaceDeleteFilesBenchmark.java
@@ -20,6 +20,7 @@
 
 import static org.apache.iceberg.types.Types.NestedField.required;
 
+import java.io.IOException;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
 import org.apache.iceberg.hadoop.HadoopTables;
@@ -73,11 +74,14 @@ public class ReplaceDeleteFilesBenchmark {
   private List<DeleteFile> deleteFiles;
   private List<DeleteFile> pendingDeleteFiles;
 
-  @Param({"50000", "100000", "500000", "1000000", "2500000"})
+  @Param({"50000", "100000", "500000", "1000000", "2000000"})
   private int numFiles;
 
+  @Param({"5", "25", "50", "100"})
+  private int percentDeleteFilesReplaced;
+
   @Setup
-  public void setupBenchmark() {
+  public void setupBenchmark() throws IOException {
     initTable();
     initFiles();
   }
@@ -104,27 +108,39 @@ private void dropTable() {
     TABLES.dropTable(TABLE_IDENT);
   }
 
-  private void initFiles() {
+  private void initFiles() throws IOException {
     List<DeleteFile> generatedDeleteFiles = Lists.newArrayListWithExpectedSize(numFiles);
     List<DeleteFile> generatedPendingDeleteFiles = Lists.newArrayListWithExpectedSize(numFiles);
 
     RowDelta rowDelta = table.newRowDelta();
+    int filesToDelete = (int) Math.ceil(numFiles * (percentDeleteFilesReplaced / 100.0));
 
     for (int ordinal = 0; ordinal < numFiles; ordinal++) {
       DataFile dataFile = FileGenerationUtil.generateDataFile(table, null);
       rowDelta.addRows(dataFile);
-
-      DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
-      rowDelta.addDeletes(deleteFile);
-      generatedDeleteFiles.add(deleteFile);
-
-      DeleteFile pendingDeleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
-      generatedPendingDeleteFiles.add(pendingDeleteFile);
+      if (filesToDelete > 0) {
+        DeleteFile deleteFile = FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
+        rowDelta.addDeletes(deleteFile);
+        generatedDeleteFiles.add(deleteFile);
+        DeleteFile pendingDeleteFile =
+            FileGenerationUtil.generatePositionDeleteFile(table, dataFile);
+        generatedPendingDeleteFiles.add(pendingDeleteFile);
+        filesToDelete--;
+      }
     }
 
     rowDelta.commit();
 
-    this.deleteFiles = generatedDeleteFiles;
+    List<DeleteFile> deleteFilesReadFromManifests = Lists.newArrayList();
+    List<ManifestFile> deleteManifests = table.currentSnapshot().deleteManifests(table.io());
+    for (ManifestFile deleteManifest : deleteManifests) {
+      try (ManifestReader<DeleteFile> manifestReader =
+          ManifestFiles.readDeleteManifest(deleteManifest, table.io(), table.specs())) {
+        manifestReader.iterator().forEachRemaining(deleteFilesReadFromManifests::add);
+      }
+    }
+
+    this.deleteFiles = deleteFilesReadFromManifests;
     this.pendingDeleteFiles = generatedPendingDeleteFiles;
   }
 }
diff --git a/gradle.properties b/gradle.properties
@@ -34,4 +34,4 @@ org.gradle.parallel=true
 org.gradle.configureondemand=true
 # explicitly disable the configuration cache
 org.gradle.configuration-cache=false
-org.gradle.jvmargs=-Xmx1024m
+org.gradle.jvmargs=-Xmx64G