Skip to content

Commit

Permalink
revert
Browse files Browse the repository at this point in the history
  • Loading branch information
ludlows committed Sep 17, 2024
1 parent 1fd8aa4 commit 0658c4f
Show file tree
Hide file tree
Showing 2 changed files with 4 additions and 4 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ protected void doRewrite(String groupId, List<PositionDeletesScanTask> group) {
.option(SparkReadOptions.SCAN_TASK_SET_ID, groupId)
.option(SparkReadOptions.SPLIT_SIZE, splitSize(inputSize(group)))
.option(SparkReadOptions.FILE_OPEN_COST, "0")
.load(table().name());
.load(groupId);

// keep only valid position deletes
Dataset<Row> dataFiles = dataFiles(partitionType, partition);
Expand All @@ -118,7 +118,7 @@ protected void doRewrite(String groupId, List<PositionDeletesScanTask> group) {
.option(SparkWriteOptions.REWRITTEN_FILE_SCAN_TASK_SET_ID, groupId)
.option(SparkWriteOptions.TARGET_DELETE_FILE_SIZE_BYTES, writeMaxFileSize())
.mode("append")
.save(table().name());
.save(groupId);
}

/** Returns entries of {@link DataFilesTable} of specified partition */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ public void doRewrite(String groupId, List<FileScanTask> group) {
.read()
.format("iceberg")
.option(SparkReadOptions.SCAN_TASK_SET_ID, groupId)
.load(table().name());
.load(groupId);

Dataset<Row> sortedDF = sortedDF(scanDF, sortFunction(group));

Expand All @@ -136,7 +136,7 @@ public void doRewrite(String groupId, List<FileScanTask> group) {
.option(SparkWriteOptions.USE_TABLE_DISTRIBUTION_AND_ORDERING, "false")
.option(SparkWriteOptions.OUTPUT_SPEC_ID, outputSpecId())
.mode("append")
.save(table().name());
.save(groupId);
}

private Function<Dataset<Row>, Dataset<Row>> sortFunction(List<FileScanTask> group) {
Expand Down

0 comments on commit 0658c4f

Please sign in to comment.