From 4d38ead91472939b5d5ea52b38737c18d6798d84 Mon Sep 17 00:00:00 2001 From: wforget <643348094@qq.com> Date: Fri, 28 Jun 2024 18:37:42 +0800 Subject: [PATCH] [VL][UNIFFLE] Remove inc BytesWritten in VeloxUniffleColumnarShuffleWriter --- .../shuffle/writer/VeloxUniffleColumnarShuffleWriter.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java index 471d70f4d1a5..b9b68fa9c024 100644 --- a/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java +++ b/gluten-uniffle/velox/src/main/java/org/apache/spark/shuffle/writer/VeloxUniffleColumnarShuffleWriter.java @@ -217,7 +217,8 @@ public Set applicablePhases() { - splitResult.getTotalWriteTime() - splitResult.getTotalCompressTime()); - shuffleWriteMetrics.incBytesWritten(splitResult.getTotalBytesWritten()); + // bytesWritten is calculated in uniffle side: WriteBufferManager.createShuffleBlock + // shuffleWriteMetrics.incBytesWritten(splitResult.getTotalBytesWritten()); shuffleWriteMetrics.incWriteTime( splitResult.getTotalWriteTime() + splitResult.getTotalPushTime()); // partitionLengths is calculate in uniffle side