Add memory reclaim max wait timeout plus spill improvement (facebooki…

…ncubator#7761)

Summary:
In Meta internal test, we found in some rare case the bug in parallel join build can get
stuck in an infinite loop. If memory arbitration decides to reclaim from this query, then
the task pause will hang forever as the parallel join build driver thread will keep running
on the thread. Once this happens, the entire worker will get stuck as eventually all the
driver threads will be blocked by the memory arbitration as the query executions will keep
asking for memory arbitration. Instead of hanging the entire worker query execution, it might
be better to provide a configurable option to time out the task pause wait during memory
reclaim. Therefore, if there is a hanging query, then the memory arbitration simply fails the
memory reclaim which in turn fails the hanging query execution.
This PR adds an option in memory arbitration to specify the max memory reclaim timeout.
The actual memory reclaim implementation can choose to respect the timeout or not. The
task memory reclaimer respect the timeout when wait for the task to pause. If timed out,
it will fail the memory reclaim on this task. We also add stats counter to report the timeout
event for monitoring so we can catch this in production if it happens.
We also add a timeout when the task memory pool aborts the task execution as the victim
task can be a hanging task as well.
Add unit test to verify and catch this behavior. We also improve the hash build memory
reservation when the build table is empty to avoid the flakiness in the test which should also
help in production.

Pull Request resolved: facebookincubator#7761

Reviewed By: mbasmanova

Differential Revision: D51650322

Pulled By: xiaoxmeng

fbshipit-source-id: 9c16b928cac2c584831520d331eb7f7fcd3e263b

velox/common/base/Counters.cpp

@@ -46,6 +46,10 @@ void registerVeloxCounters() {
   // Track memory reclaim bytes.
   REPORT_ADD_STAT_EXPORT_TYPE(
       kCounterMemoryReclaimedBytes, facebook::velox::StatType::SUM);
+
+  // Track the number of times that the memory reclaim wait timeouts.
+  REPORT_ADD_STAT_EXPORT_TYPE(
+      kCounterMemoryReclaimWaitTimeoutCount, facebook::velox::StatType::SUM);
 }
 
 } // namespace facebook::velox

velox/common/base/Counters.h

@@ -20,7 +20,7 @@
 
 namespace facebook::velox {
 
-// Velox Counter Registration
+/// Velox Counter Registration
 void registerVeloxCounters();
 
 constexpr folly::StringPiece kCounterHiveFileHandleGenerateLatencyMs{
@@ -39,4 +39,7 @@ constexpr folly::StringPiece kCounterMemoryReclaimedBytes{
 
 constexpr folly::StringPiece kCounterMemoryReclaimWaitTimeMs{
     "velox.memory_reclaim_wait_ms"};
+
+constexpr folly::StringPiece kCounterMemoryReclaimWaitTimeoutCount{
+    "velox.memory_reclaim_wait_timeout_count"};
 } // namespace facebook::velox

velox/common/memory/Memory.cpp

@@ -35,6 +35,7 @@ MemoryManager::MemoryManager(const MemoryManagerOptions& options)
            .capacity = std::min(options.queryMemoryCapacity, options.capacity),
            .memoryPoolInitCapacity = options.memoryPoolInitCapacity,
            .memoryPoolTransferCapacity = options.memoryPoolTransferCapacity,
+           .memoryReclaimWaitMs = options.memoryReclaimWaitMs,
            .arbitrationStateCheckCb = options.arbitrationStateCheckCb})),
       alignment_(std::max(MemoryAllocator::kMinAlignment, options.alignment)),
       checkUsageLeak_(options.checkUsageLeak),

velox/common/memory/Memory.h

@@ -114,6 +114,11 @@ struct MemoryManagerOptions {
   /// during the memory arbitration.
   uint64_t memoryPoolTransferCapacity{32 << 20};
 
+  /// Specifies the max time to wait for memory reclaim by arbitration. The
+  /// memory reclaim might fail if the max wait time has exceeded. If it is
+  /// zero, then there is no timeout.
+  uint64_t memoryReclaimWaitMs{0};
+
   /// Provided by the query system to validate the state after a memory pool
   /// enters arbitration if not null. For instance, Prestissimo provides
   /// callback to check if a memory arbitration request is issued from a driver

velox/common/memory/MemoryArbitrator.cpp

@@ -196,8 +196,11 @@ bool MemoryReclaimer::reclaimableBytes(
   return reclaimable;
 }
 
-uint64_t
-MemoryReclaimer::reclaim(MemoryPool* pool, uint64_t targetBytes, Stats& stats) {
+uint64_t MemoryReclaimer::reclaim(
+    MemoryPool* pool,
+    uint64_t targetBytes,
+    uint64_t maxWaitMs,
+    Stats& stats) {
   if (pool->kind() == MemoryPool::Kind::kLeaf) {
     return 0;
   }
@@ -230,7 +233,7 @@ MemoryReclaimer::reclaim(MemoryPool* pool, uint64_t targetBytes, Stats& stats) {
 
   uint64_t reclaimedBytes{0};
   for (const auto& candidate : candidates) {
-    const auto bytes = candidate.pool->reclaim(targetBytes, stats);
+    const auto bytes = candidate.pool->reclaim(targetBytes, maxWaitMs, stats);
     reclaimedBytes += bytes;
     if (targetBytes != 0) {
       if (bytes >= targetBytes) {

velox/common/memory/MemoryArbitrator.h

@@ -63,6 +63,13 @@ class MemoryArbitrator {
     /// during the memory arbitration.
     uint64_t memoryPoolTransferCapacity{32 << 20};
 
+    /// Specifies the max time to wait for memory reclaim by arbitration. The
+    /// memory reclaim might fail if the max time has exceeded. This prevents
+    /// the memory arbitration from getting stuck when the memory reclaim waits
+    /// for a hanging query task to pause. If it is zero, then there is no
+    /// timeout.
+    uint64_t memoryReclaimWaitMs{0};
+
     /// Provided by the query system to validate the state after a memory pool
     /// enters arbitration if not null. For instance, Prestissimo provides
     /// callback to check if a memory arbitration request is issued from a
@@ -223,11 +230,13 @@ class MemoryArbitrator {
       : capacity_(config.capacity),
         memoryPoolInitCapacity_(config.memoryPoolInitCapacity),
         memoryPoolTransferCapacity_(config.memoryPoolTransferCapacity),
+        memoryReclaimWaitMs_(config.memoryReclaimWaitMs),
         arbitrationStateCheckCb_(config.arbitrationStateCheckCb) {}
 
   const uint64_t capacity_;
   const uint64_t memoryPoolInitCapacity_;
   const uint64_t memoryPoolTransferCapacity_;
+  const uint64_t memoryReclaimWaitMs_;
   const MemoryArbitrationStateCheckCB arbitrationStateCheckCb_;
 };
 
@@ -312,10 +321,18 @@ class MemoryReclaimer {
   /// Invoked by the memory arbitrator to reclaim from memory 'pool' with
   /// specified 'targetBytes'. It is expected to reclaim at least that amount of
   /// memory bytes but there is no guarantees. If 'targetBytes' is zero, then it
-  /// reclaims all the reclaimable memory from the memory 'pool'. The function
-  /// returns the actual reclaimed memory bytes.
-  virtual uint64_t
-  reclaim(MemoryPool* pool, uint64_t targetBytes, Stats& stats);
+  /// reclaims all the reclaimable memory from the memory 'pool'. 'maxWaitMs'
+  /// specifies the max time to wait for reclaim if not zero. The memory
+  /// reclaim might fail if exceeds the timeout. The function returns the actual
+  /// reclaimed memory bytes.
+  ///
+  /// NOTE: 'maxWaitMs' is optional and the actual memory reclaim implementation
+  /// can choose to respect this timeout or not on its own.
+  virtual uint64_t reclaim(
+      MemoryPool* pool,
+      uint64_t targetBytes,
+      uint64_t maxWaitMs,
+      Stats& stats);
 
   /// Invoked by the memory arbitrator to abort memory 'pool' and the associated
   /// query execution when encounters non-recoverable memory reclaim error or

velox/common/memory/MemoryPool.cpp

@@ -935,11 +935,12 @@ bool MemoryPoolImpl::reclaimableBytes(uint64_t& reclaimableBytes) const {
 
 uint64_t MemoryPoolImpl::reclaim(
     uint64_t targetBytes,
+    uint64_t maxWaitMs,
     memory::MemoryReclaimer::Stats& stats) {
   if (reclaimer() == nullptr) {
     return 0;
   }
-  return reclaimer()->reclaim(this, targetBytes, stats);
+  return reclaimer()->reclaim(this, targetBytes, maxWaitMs, stats);
 }
 
 void MemoryPoolImpl::enterArbitration() {

velox/common/memory/MemoryPool.h

@@ -410,10 +410,13 @@ class MemoryPool : public std::enable_shared_from_this<MemoryPool> {
   /// with specified reclaim target bytes. If 'targetBytes' is zero, then it
   /// tries to reclaim all the reclaimable memory from the memory pool. It is
   /// noop if the reclaimer is not set, otherwise invoke the reclaimer's
-  /// corresponding method. The function returns the actually freed capacity
-  /// from the root of this memory pool.
+  /// corresponding method. If not zero, 'maxWaitMs' specifies the max time in
+  /// milliseconds to wait for reclaim. The memory reclaim might fail if exceeds
+  /// the timeout. The function returns the actually freed capacity from the
+  /// root of this memory pool.
   virtual uint64_t reclaim(
       uint64_t targetBytes,
+      uint64_t maxWaitMs,
       memory::MemoryReclaimer::Stats& stats) = 0;
 
   /// Invoked by the memory arbitrator to abort a root memory pool. The function
@@ -636,8 +639,10 @@ class MemoryPoolImpl : public MemoryPool {
 
   bool reclaimableBytes(uint64_t& reclaimableBytes) const override;
 
-  uint64_t reclaim(uint64_t targetBytes, memory::MemoryReclaimer::Stats& stats)
-      override;
+  uint64_t reclaim(
+      uint64_t targetBytes,
+      uint64_t maxWaitMs,
+      memory::MemoryReclaimer::Stats& stats) override;
 
   uint64_t shrink(uint64_t targetBytes = 0) override;
 

velox/common/memory/tests/MemoryArbitratorTest.cpp

@@ -323,11 +323,11 @@ TEST_F(MemoryReclaimerTest, common) {
       uint64_t reclaimableBytes;
       ASSERT_FALSE(pool->reclaimableBytes(reclaimableBytes));
       ASSERT_EQ(reclaimableBytes, 0);
-      ASSERT_EQ(pool->reclaim(0, stats_), 0);
+      ASSERT_EQ(pool->reclaim(0, 0, stats_), 0);
       ASSERT_EQ(stats_, MemoryReclaimer::Stats{});
-      ASSERT_EQ(pool->reclaim(100, stats_), 0);
+      ASSERT_EQ(pool->reclaim(100, 0, stats_), 0);
       ASSERT_EQ(stats_, MemoryReclaimer::Stats{});
-      ASSERT_EQ(pool->reclaim(kMaxMemory, stats_), 0);
+      ASSERT_EQ(pool->reclaim(kMaxMemory, 0, stats_), 0);
       ASSERT_EQ(stats_, MemoryReclaimer::Stats{});
     }
     ASSERT_EQ(stats_, MemoryReclaimer::Stats{});
@@ -356,6 +356,7 @@ class MockLeafMemoryReclaimer : public MemoryReclaimer {
   uint64_t reclaim(
       MemoryPool* /*unused*/,
       uint64_t targetBytes,
+      uint64_t /*unused*/,
       Stats& stats) noexcept override {
     std::lock_guard<std::mutex> l(mu_);
     uint64_t reclaimedBytes{0};
@@ -445,7 +446,7 @@ TEST_F(MemoryReclaimerTest, mockReclaim) {
   const int numReclaims = 5;
   const int numBytesToReclaim = allocBytes * 3;
   for (int iter = 0; iter < numReclaims; ++iter) {
-    const auto reclaimedBytes = root->reclaim(numBytesToReclaim, stats_);
+    const auto reclaimedBytes = root->reclaim(numBytesToReclaim, 0, stats_);
     ASSERT_EQ(reclaimedBytes, numBytesToReclaim);
     ASSERT_EQ(reclaimedBytes, stats_.reclaimedBytes);
     ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));
@@ -454,12 +455,12 @@ TEST_F(MemoryReclaimerTest, mockReclaim) {
   }
   ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));
   ASSERT_EQ(totalUsedBytes, reclaimableBytes);
-  ASSERT_EQ(root->reclaim(allocBytes + 1, stats_), 2 * allocBytes);
-  ASSERT_EQ(root->reclaim(allocBytes - 1, stats_), allocBytes);
+  ASSERT_EQ(root->reclaim(allocBytes + 1, 0, stats_), 2 * allocBytes);
+  ASSERT_EQ(root->reclaim(allocBytes - 1, 0, stats_), allocBytes);
   ASSERT_EQ(3 * allocBytes, stats_.reclaimedBytes);
 
   const uint64_t expectedReclaimedBytes = totalUsedBytes;
-  ASSERT_EQ(root->reclaim(0, stats_), expectedReclaimedBytes);
+  ASSERT_EQ(root->reclaim(0, 0, stats_), expectedReclaimedBytes);
   ASSERT_EQ(3 * allocBytes + expectedReclaimedBytes, stats_.reclaimedBytes);
   ASSERT_EQ(totalUsedBytes, 0);
   ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));
@@ -497,7 +498,7 @@ TEST_F(MemoryReclaimerTest, mockReclaimMoreThanAvailable) {
   ASSERT_EQ(reclaimableBytes, totalUsedBytes);
   const uint64_t expectedReclaimedBytes = totalUsedBytes;
   ASSERT_EQ(
-      root->reclaim(totalUsedBytes + 100, stats_), expectedReclaimedBytes);
+      root->reclaim(totalUsedBytes + 100, 0, stats_), expectedReclaimedBytes);
   ASSERT_EQ(expectedReclaimedBytes, stats_.reclaimedBytes);
   ASSERT_EQ(totalUsedBytes, 0);
   ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));
@@ -558,7 +559,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child.
   // So expected reclaimable allocation units are {10, 11, 8, *14*, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, 0, stats_),
       2 * allocUnitBytes);
   ASSERT_EQ(2 * allocUnitBytes, stats_.reclaimedBytes);
   totalAllocUnits -= 2;
@@ -568,7 +569,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child.
   // So expected reclaimable allocation units are {10, 11, 8, *12*, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, 0, stats_),
       2 * allocUnitBytes);
   ASSERT_EQ(4 * allocUnitBytes, stats_.reclaimedBytes);
   totalAllocUnits -= 2;
@@ -578,7 +579,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child.
   // So expected reclaimable allocation units are {10, 11, 8, *4*, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 8 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 8 * allocUnitBytes, 0, stats_),
       8 * allocUnitBytes);
   ASSERT_EQ(12 * allocUnitBytes, stats_.reclaimedBytes);
   totalAllocUnits -= 8;
@@ -588,7 +589,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child.
   // So expected reclaimable allocation gunits are {10, *9*, 8, 4, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 2 * allocUnitBytes, 0, stats_),
       2 * allocUnitBytes);
   totalAllocUnits -= 2;
   verify({10, 9, 8, 4, 5});
@@ -597,7 +598,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child.
   // So expected reclaimable allocation units are {*7*, 9, 8, 4, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 3 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 3 * allocUnitBytes, 0, stats_),
       3 * allocUnitBytes);
   totalAllocUnits -= 3;
   verify({7, 9, 8, 4, 5});
@@ -606,7 +607,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child and two from 2nd child.
   // So expected reclaimable allocation units are {7, *0*, *6*, 4, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 11 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 11 * allocUnitBytes, 0, stats_),
       11 * allocUnitBytes);
   totalAllocUnits -= 11;
   verify({7, 0, 6, 4, 5});
@@ -615,7 +616,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child and three from 2nd child.
   // So expected reclaimable allocation units are {*0*, 0, *3*, 4, 5}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 10 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 10 * allocUnitBytes, 0, stats_),
       10 * allocUnitBytes);
   totalAllocUnits -= 10;
   verify({0, 0, 3, 4, 5});
@@ -624,7 +625,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // child and 4 from 4th child and 1 from 2nd.
   // So expected reclaimable allocation units are {0, 0, 2, *0*, *0*}
   ASSERT_EQ(
-      root->reclaimer()->reclaim(root.get(), 10 * allocUnitBytes, stats_),
+      root->reclaimer()->reclaim(root.get(), 10 * allocUnitBytes, 0, stats_),
       10 * allocUnitBytes);
   totalAllocUnits -= 10;
   verify({0, 0, 2, 0, 0});
@@ -633,7 +634,7 @@ TEST_F(MemoryReclaimerTest, orderedReclaim) {
   // cleared.
   ASSERT_EQ(
       root->reclaimer()->reclaim(
-          root.get(), totalAllocUnits * allocUnitBytes, stats_),
+          root.get(), totalAllocUnits * allocUnitBytes, 0, stats_),
       totalAllocUnits * allocUnitBytes);
   totalAllocUnits = 0;
   verify({0, 0, 0, 0, 0});
@@ -722,7 +723,7 @@ TEST_F(MemoryReclaimerTest, concurrentRandomMockReclaims) {
           bytesToReclaim = 0;
         }
       }
-      const auto reclaimedBytes = root->reclaim(bytesToReclaim, stats_);
+      const auto reclaimedBytes = root->reclaim(bytesToReclaim, 0, stats_);
       totalReclaimedBytes += reclaimedBytes;
       if (reclaimedBytes < bytesToReclaim) {
         ASSERT_GT(bytesToReclaim, oldUsedBytes);
@@ -761,7 +762,7 @@ TEST_F(MemoryReclaimerTest, concurrentRandomMockReclaims) {
   ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));
   ASSERT_EQ(reclaimableBytes, totalUsedBytes);
 
-  root->reclaim(0, stats_);
+  root->reclaim(0, 0, stats_);
   ASSERT_EQ(totalReclaimedBytes + reclaimableBytes, stats_.reclaimedBytes);
 
   ASSERT_TRUE(root->reclaimableBytes(reclaimableBytes));

velox/common/memory/tests/MemoryPoolTest.cpp

@@ -2817,9 +2817,9 @@ TEST_P(MemoryPoolTest, reclaimAPIsWithDefaultReclaimer) {
       uint64_t reclaimableBytes{100};
       ASSERT_FALSE(pool->reclaimableBytes(reclaimableBytes));
       ASSERT_EQ(reclaimableBytes, 0);
-      ASSERT_EQ(pool->reclaim(0, stats_), 0);
-      ASSERT_EQ(pool->reclaim(100, stats_), 0);
-      ASSERT_EQ(pool->reclaim(kMaxMemory, stats_), 0);
+      ASSERT_EQ(pool->reclaim(0, 0, stats_), 0);
+      ASSERT_EQ(pool->reclaim(100, 0, stats_), 0);
+      ASSERT_EQ(pool->reclaim(kMaxMemory, 0, stats_), 0);
     }
     for (const auto& allocation : allocations) {
       allocation.pool->free(allocation.buffer, allocation.size);

velox/common/memory/tests/MockSharedArbitratorTest.cpp

@@ -161,8 +161,11 @@ class MockMemoryOperator {
       return op_->reclaimableBytes(pool, reclaimableBytes);
     }
 
-    uint64_t reclaim(MemoryPool* pool, uint64_t targetBytes, Stats& stats)
-        override {
+    uint64_t reclaim(
+        MemoryPool* pool,
+        uint64_t targetBytes,
+        uint64_t /*unused*/,
+        Stats& stats) override {
       ++numReclaims_;
       if (!reclaimable_) {
         return 0;

velox/connectors/hive/HiveDataSink.cpp

@@ -868,6 +868,7 @@ bool HiveDataSink::WriterReclaimer::reclaimableBytes(
 uint64_t HiveDataSink::WriterReclaimer::reclaim(
     memory::MemoryPool* pool,
     uint64_t targetBytes,
+    uint64_t maxWaitMs,
     memory::MemoryReclaimer::Stats& stats) {
   VELOX_CHECK_EQ(pool->name(), writerInfo_->writerPool->name());
   if (!dataSink_->canReclaim()) {
@@ -887,7 +888,7 @@ uint64_t HiveDataSink::WriterReclaimer::reclaim(
   const uint64_t memoryUsageBeforeReclaim = pool->currentBytes();
   const std::string memoryUsageTreeBeforeReclaim = pool->treeMemoryUsage();
   const auto reclaimedBytes =
-      exec::MemoryReclaimer::reclaim(pool, targetBytes, stats);
+      exec::MemoryReclaimer::reclaim(pool, targetBytes, maxWaitMs, stats);
   const uint64_t memoryUsageAfterReclaim = pool->currentBytes();
   if (memoryUsageAfterReclaim > memoryUsageBeforeReclaim) {
     VELOX_FAIL(

velox/connectors/hive/HiveDataSink.h

@@ -454,6 +454,7 @@ class HiveDataSink : public DataSink {
     uint64_t reclaim(
         memory::MemoryPool* pool,
         uint64_t targetBytes,
+        uint64_t maxWaitMs,
         memory::MemoryReclaimer::Stats& stats) override;
 
    private:

velox/connectors/hive/tests/HiveDataSinkTest.cpp

@@ -671,15 +671,15 @@ TEST_F(HiveDataSinkTest, memoryReclaim) {
     if (testData.expectedWriterReclaimed) {
       ASSERT_TRUE(root_->reclaimableBytes(reclaimableBytes));
       ASSERT_GT(reclaimableBytes, 0);
-      ASSERT_GT(root_->reclaim(256L << 20, stats), 0);
+      ASSERT_GT(root_->reclaim(256L << 20, 0, stats), 0);
       ASSERT_GT(stats.reclaimExecTimeUs, 0);
       ASSERT_GT(stats.reclaimedBytes, 0);
       // We expect dwrf writer set numNonReclaimableAttempts counter.
       ASSERT_LE(stats.numNonReclaimableAttempts, 1);
     } else {
       ASSERT_FALSE(root_->reclaimableBytes(reclaimableBytes));
       ASSERT_EQ(reclaimableBytes, 0);
-      ASSERT_EQ(root_->reclaim(256L << 20, stats), 0);
+      ASSERT_EQ(root_->reclaim(256L << 20, 0, stats), 0);
       ASSERT_EQ(stats.reclaimExecTimeUs, 0);
       ASSERT_EQ(stats.reclaimedBytes, 0);
       if (testData.expectedWriterReclaimEnabled) {
@@ -830,7 +830,7 @@ TEST_F(HiveDataSinkTest, memoryReclaimAfterClose) {
       ASSERT_FALSE(root_->reclaimableBytes(reclaimableBytes));
       ASSERT_EQ(reclaimableBytes, 0);
     }
-    ASSERT_EQ(root_->reclaim(1L << 30, stats), 0);
+    ASSERT_EQ(root_->reclaim(1L << 30, 0, stats), 0);
     ASSERT_EQ(stats.reclaimExecTimeUs, 0);
     ASSERT_EQ(stats.reclaimedBytes, 0);
     if (testData.expectedWriterReclaimEnabled) {