From 73fb02e271e5357eec459e6562a4b9449ba86571 Mon Sep 17 00:00:00 2001 From: Jialiang Tan Date: Tue, 11 Jun 2024 20:58:46 -0700 Subject: [PATCH] Fix writer OOM by reducing initial flush buffer reservation (#10147) Summary: Initial memory reservation is based on the tracked ratio maintained by RatioTracker. The one for flush is used for computing memory to reserve before flushing. For initial ratio it was default to 1.0f which ends up writer to reserve >2x (reservation headroom * ratio for flush) of raw memory from writes. This default ratio is used in the first iteration and is too large. Reduced it to 0.1 for better memory utilization. This helps to reduce in many cases writer memory usage by more than 60%. Pull Request resolved: https://github.com/facebookincubator/velox/pull/10147 Reviewed By: xiaoxmeng Differential Revision: D58440137 Pulled By: tanjialiang fbshipit-source-id: c715f99e9a081cc1947cde4f2d1b9c9fbe020b50 --- velox/dwio/dwrf/test/RatioTrackerTest.cpp | 2 +- velox/dwio/dwrf/writer/RatioTracker.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/velox/dwio/dwrf/test/RatioTrackerTest.cpp b/velox/dwio/dwrf/test/RatioTrackerTest.cpp index 912da20c5f23..4f4bddf0662f 100644 --- a/velox/dwio/dwrf/test/RatioTrackerTest.cpp +++ b/velox/dwio/dwrf/test/RatioTrackerTest.cpp @@ -53,7 +53,7 @@ TEST(RatioTrackerTest, BasicTests) { {{10, 11}, {20, 1}, {30, 1}, {40, 1}, {50, 1}}, 0.1f}, TestCase{std::make_shared(), {}, 0.3f}, - TestCase{std::make_shared(), {}, 1.0f}, + TestCase{std::make_shared(), {}, 0.1f}, TestCase{std::make_shared(), {}, 0.0f}, }; diff --git a/velox/dwio/dwrf/writer/RatioTracker.h b/velox/dwio/dwrf/writer/RatioTracker.h index 2152396843a7..38c50ccadafa 100644 --- a/velox/dwio/dwrf/writer/RatioTracker.h +++ b/velox/dwio/dwrf/writer/RatioTracker.h @@ -19,7 +19,7 @@ #include constexpr float kCompressionRatioInitialGuess = 0.3f; -constexpr float kFlushOverheadRatioInitialGuess = 1.0f; +constexpr float kFlushOverheadRatioInitialGuess = 0.1f; // TODO(T79660637): make write cost estimate more granular. constexpr float kAverageRowSizeInitialGuess = 0.0f;