Add cache error message for all memory pool allocations (facebookincu…

…bator#7499) Summary: Pull Request resolved: facebookincubator#7499 Reviewed By: oerling Differential Revision: D51167510 Pulled By: xiaoxmeng fbshipit-source-id: a2b7893db77ce89e36e40e8b5fc5d181b17b64d8
zhli1142015 · Nov 10, 2023 · 71568ff · 71568ff
1 parent 3f30748
commit 71568ff
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 28 deletions.
diff --git a/velox/common/caching/AsyncDataCache.cpp b/velox/common/caching/AsyncDataCache.cpp
@@ -671,8 +671,8 @@ bool AsyncDataCache::makeSpace(
       sizeMultiplier *= 2;
     }
   }
-  memory::setCacheFailureMessage(fmt::format(
-      "After failing to evict from cache state: {}", toString(false)));
+  memory::setCacheFailureMessage(
+      fmt::format("Failed to evict from cache state: {}", toString(false)));
   return false;
 }
 

diff --git a/velox/common/memory/MemoryPool.cpp b/velox/common/memory/MemoryPool.cpp
@@ -430,10 +430,11 @@ void* MemoryPoolImpl::allocate(int64_t size) {
   if (FOLLY_UNLIKELY(buffer == nullptr)) {
     release(alignedSize);
     VELOX_MEM_ALLOC_ERROR(fmt::format(
-        "{} failed with {} from {}",
+        "{} failed with {} from {} {}",
         __FUNCTION__,
         succinctBytes(size),
-        toString()));
+        toString(),
+        allocator_->getAndClearFailureMessage()));
   }
   DEBUG_RECORD_ALLOC(buffer, size);
   return buffer;
@@ -448,11 +449,12 @@ void* MemoryPoolImpl::allocateZeroFilled(int64_t numEntries, int64_t sizeEach) {
   if (FOLLY_UNLIKELY(buffer == nullptr)) {
     release(alignedSize);
     VELOX_MEM_ALLOC_ERROR(fmt::format(
-        "{} failed with {} entries and {} each from {}",
+        "{} failed with {} entries and {} each from {} {}",
         __FUNCTION__,
         numEntries,
         succinctBytes(sizeEach),
-        toString()));
+        toString(),
+        allocator_->getAndClearFailureMessage()));
   }
   DEBUG_RECORD_ALLOC(buffer, size);
   return buffer;
@@ -467,11 +469,12 @@ void* MemoryPoolImpl::reallocate(void* p, int64_t size, int64_t newSize) {
   if (FOLLY_UNLIKELY(newP == nullptr)) {
     release(alignedNewSize);
     VELOX_MEM_ALLOC_ERROR(fmt::format(
-        "{} failed with new {} and old {} from {}",
+        "{} failed with new {} and old {} from {} {}",
         __FUNCTION__,
         succinctBytes(newSize),
         succinctBytes(size),
-        toString()));
+        toString(),
+        allocator_->getAndClearFailureMessage()));
   }
   DEBUG_RECORD_ALLOC(newP, newSize);
   if (p != nullptr) {

diff --git a/velox/common/memory/tests/MemoryPoolTest.cpp b/velox/common/memory/tests/MemoryPoolTest.cpp
@@ -588,27 +588,53 @@ TEST_P(MemoryPoolTest, MemoryCapExceptions) {
         ASSERT_EQ(error_code::kMemAllocError.c_str(), ex.errorCode());
         ASSERT_TRUE(ex.isRetriable());
         if (useMmap_) {
-          ASSERT_EQ(
-              fmt::format(
-                  "allocate failed with 128.00MB from Memory Pool["
-                  "static_quota LEAF root[MemoryCapExceptions] "
-                  "parent[MemoryCapExceptions] MMAP track-usage {}]<max "
-                  "capacity 256.00MB capacity 256.00MB used 0B available 0B "
-                  "reservation [used 0B, reserved 0B, min 0B] counters [allocs "
-                  "1, frees 0, reserves 0, releases 0, collisions 0])>",
-                  isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
-              ex.message());
+          if (useCache_) {
+            ASSERT_EQ(
+                fmt::format(
+                    "allocate failed with 128.00MB from Memory Pool["
+                    "static_quota LEAF root[MemoryCapExceptions] "
+                    "parent[MemoryCapExceptions] MMAP track-usage {}]<max "
+                    "capacity 256.00MB capacity 256.00MB used 0B available 0B "
+                    "reservation [used 0B, reserved 0B, min 0B] counters [allocs "
+                    "1, frees 0, reserves 0, releases 0, collisions 0])> Failed to evict from cache state: AsyncDataCache:\nCache size: 0B tinySize: 0B large size: 0B\nCache entries: 0 read pins: 0 write pins: 0 pinned shared: 0B pinned exclusive: 0B\n num write wait: 0 empty entries: 0\nCache access miss: 0 hit: 0 hit bytes: 0B eviction: 0 eviction checks: 0\nPrefetch entries: 0 bytes: 0B\nAlloc Megaclocks 0\nAllocated pages: 0 cached pages: 0\n",
+                    isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
+                ex.message());
+          } else {
+            ASSERT_EQ(
+                fmt::format(
+                    "allocate failed with 128.00MB from Memory Pool["
+                    "static_quota LEAF root[MemoryCapExceptions] "
+                    "parent[MemoryCapExceptions] MMAP track-usage {}]<max "
+                    "capacity 256.00MB capacity 256.00MB used 0B available 0B "
+                    "reservation [used 0B, reserved 0B, min 0B] counters [allocs "
+                    "1, frees 0, reserves 0, releases 0, collisions 0])> ",
+                    isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
+                ex.message());
+          }
         } else {
-          ASSERT_EQ(
-              fmt::format(
-                  "allocate failed with 128.00MB from Memory Pool"
-                  "[static_quota LEAF root[MemoryCapExceptions] "
-                  "parent[MemoryCapExceptions] MALLOC track-usage {}]"
-                  "<max capacity 256.00MB capacity 256.00MB used 0B available "
-                  "0B reservation [used 0B, reserved 0B, min 0B] counters "
-                  "[allocs 1, frees 0, reserves 0, releases 0, collisions 0])>",
-                  isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
-              ex.message());
+          if (useCache_) {
+            ASSERT_EQ(
+                fmt::format(
+                    "allocate failed with 128.00MB from Memory Pool"
+                    "[static_quota LEAF root[MemoryCapExceptions] "
+                    "parent[MemoryCapExceptions] MALLOC track-usage {}]"
+                    "<max capacity 256.00MB capacity 256.00MB used 0B available "
+                    "0B reservation [used 0B, reserved 0B, min 0B] counters "
+                    "[allocs 1, frees 0, reserves 0, releases 0, collisions 0])> Failed to evict from cache state: AsyncDataCache:\nCache size: 0B tinySize: 0B large size: 0B\nCache entries: 0 read pins: 0 write pins: 0 pinned shared: 0B pinned exclusive: 0B\n num write wait: 0 empty entries: 0\nCache access miss: 0 hit: 0 hit bytes: 0B eviction: 0 eviction checks: 0\nPrefetch entries: 0 bytes: 0B\nAlloc Megaclocks 0\nAllocated pages: 0 cached pages: 0\n",
+                    isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
+                ex.message());
+          } else {
+            ASSERT_EQ(
+                fmt::format(
+                    "allocate failed with 128.00MB from Memory Pool"
+                    "[static_quota LEAF root[MemoryCapExceptions] "
+                    "parent[MemoryCapExceptions] MALLOC track-usage {}]"
+                    "<max capacity 256.00MB capacity 256.00MB used 0B available "
+                    "0B reservation [used 0B, reserved 0B, min 0B] counters "
+                    "[allocs 1, frees 0, reserves 0, releases 0, collisions 0])> ",
+                    isLeafThreadSafe_ ? "thread-safe" : "non-thread-safe"),
+                ex.message());
+          }
         }
       }
     }