making getResource and getNumPending private

src/umpire/Umpire.cpp

@@ -289,6 +289,8 @@ util::AllocationRecord deregister_external_allocation(void* ptr)
 
 Resource getResource(Allocator a, void* ptr)
 {
+  UMPIRE_LOG(Warning, "This function will return a generic Camp resource which is not comparable to a specific Camp resource!");
+
   auto s = a.getAllocationStrategy();
   strategy::ResourceAwarePool* rap{dynamic_cast<strategy::ResourceAwarePool*>(s)};
 
@@ -299,7 +301,7 @@ Resource getResource(Allocator a, void* ptr)
   return rap->getResource(ptr);
 }
 
-std::size_t getPendingSize(Allocator a)
+std::size_t getNumPending(Allocator a)
 {
   auto s = a.getAllocationStrategy();
   strategy::ResourceAwarePool* rap{dynamic_cast<strategy::ResourceAwarePool*>(s)};
@@ -308,7 +310,7 @@ std::size_t getPendingSize(Allocator a)
     UMPIRE_ERROR(runtime_error, fmt::format("Allocator \"{}\" is not a ResourceAwarePool!", a.getName()));
   }
 
-  return rap->getPendingSize();
+  return rap->getNumPending();
 }
 
 bool try_coalesce(Allocator a)

src/umpire/Umpire.hpp

@@ -194,6 +194,7 @@ util::AllocationRecord deregister_external_allocation(void* ptr);
 
 /*!
  * \brief Returns the Camp resource associated with a particular allocation
+ * **This function is used mainly for testing purposes.**
  *
  * \param Umpire allocator which was used to allocate the data
  * \param Pointer which was used for the allocation
@@ -204,15 +205,14 @@ util::AllocationRecord deregister_external_allocation(void* ptr);
 Resource getResource(Allocator a, void* ptr);
 
 /*!
- * \brief Returns the current number of pending chunks associated with the pool. The
- * pending chunks are those that have been scheduled to deallocate but may or may not
- * have actually been deallocated yet.
+ * \brief Returns the number of pending chunks associated with a ResourceAwarePool Allocator
+ * **This function is used mainly for testing purposes.**
  *
- * \param Umpire allocator with ResourceAwarePool strategy
+ * \param Umpire ResourceAwarePool allocator
  *
- * \return Number of pending chunks in the ResourceAwarePool
+ * \return Number of currently pending chunks in the ResourceAwarePool
  */
-std::size_t getPendingSize(Allocator a);
+std::size_t getNumPending(Allocator a);
 
 /*!
  * \brief Attempt to coalesce Allocator a, return true if a coalesce was performed.

src/umpire/strategy/ResourceAwarePool.cpp

@@ -365,7 +365,7 @@ std::size_t ResourceAwarePool::getTotalBlocks() const noexcept
   return m_total_blocks;
 }
 
-std::size_t ResourceAwarePool::getPendingSize() const noexcept
+std::size_t ResourceAwarePool::getNumPending() const noexcept
 {
   return m_pending_map.size();
 }

src/umpire/strategy/ResourceAwarePool.hpp

@@ -20,6 +20,7 @@
 #include "umpire/strategy/mixins/AlignedAllocation.hpp"
 #include "umpire/util/FixedMallocPool.hpp"
 #include "umpire/util/MemoryResourceTraits.hpp"
+#include "umpire/Umpire.hpp"
 
 using Resource = camp::resources::Resource;
 using Event = camp::resources::Event;
@@ -70,14 +71,23 @@ class ResourceAwarePool : public AllocationStrategy, private mixins::AlignedAllo
 
   ResourceAwarePool(const ResourceAwarePool&) = delete;
 
+  // Granting the gtest function access to private methods below
+  friend camp::resources::Resource umpire::getResource(Allocator a, void* ptr);
+  friend std::size_t umpire::getNumPending(Allocator a);
+
+  // Allocate function needed since we inherit from AllocationStrategy.
+  // If this method is called, it will be an error (Need to call allocate_resource
+  // with a Camp resource instead).
   void* allocate(std::size_t bytes) override;
+
   /*!
    * \brief Allocate memory with the ResourceAwarePool
    *
    * \param r The Camp resource that will own the memory
    * \param bytes The size in bytes for the allocation
    */
   void* allocate_resource(Resource r, std::size_t bytes) override;
+
   /*!
    * \brief Deallocate memory with the ResourceAwarePool
    *
@@ -86,30 +96,22 @@ class ResourceAwarePool : public AllocationStrategy, private mixins::AlignedAllo
    * \param bytes The size in bytes for the allocation
    */
   void deallocate_resource(Resource r, void* ptr, std::size_t size) override;
+
+  /*!
+   * \brief Deallocate function will call private getResource function
+   * to get the resource associated with teh pointer and then call deallocate_resource
+   * above.
+   */
   void deallocate(void* ptr, std::size_t size) override;
   void release() override;
 
   std::size_t getActualSize() const noexcept override;
   std::size_t getCurrentSize() const noexcept override;
   std::size_t getReleasableSize() const noexcept;
   std::size_t getActualHighwaterMark() const noexcept;
-  std::size_t getPendingSize() const noexcept;
-
-  int getNumUsed();
-  int getNumFree();
-  int getNumPending();
-  void printFree();
-  void printUsed();
-  void printPending();
 
   Platform getPlatform() noexcept override;
-  /*!
-   * \brief get the (generic) camp resource associated with a ptr
-   */
-  Resource getResource(void* ptr) const;
-
   MemoryResourceTraits getTraits() const noexcept override;
-
   bool tracksMemoryUse() const noexcept override;
 
   /*!
@@ -168,12 +170,23 @@ class ResourceAwarePool : public AllocationStrategy, private mixins::AlignedAllo
   };
 
   using PointerMap = std::unordered_map<void*, Chunk*>;
-  // using PendingMap = std::unordered_map<camp::resources::Resource, Chunk*>; //Should this be a vector of chunks? list
-  // of chunks?
   using PendingMap = std::vector<Chunk*>;
   using SizeMap =
       std::multimap<std::size_t, Chunk*, std::less<std::size_t>, pool_allocator<std::pair<const std::size_t, Chunk*>>>;
 
+ protected:
+  /*!
+   * \brief Get the camp resource associated with a ptr
+   *
+   * \param ptr The pointer to data allocated with a ResourceAwarePool
+   */
+  Resource getResource(void* ptr) const;
+
+  /*!
+   * \brief Get the number of Pending chunks in the pool.
+   */
+  std::size_t getNumPending() const noexcept;
+
  public:
   struct Chunk {
     Chunk(void* ptr, std::size_t s, std::size_t cs, Resource r) : data{ptr}, size{s}, chunk_size{cs}, m_resource{r}

tests/integration/resource_aware_pool_tests.cpp

@@ -17,6 +17,21 @@
 
 using namespace camp::resources;
 
+#if defined(UMPIRE_ENABLE_CUDA)
+  using resource_type = Cuda;
+#elif defined(UMPIRE_ENABLE_HIP)
+  using resource_type = Hip;
+#endif
+
+std::string unique_name()
+{
+  static int unique_name_id{0};
+  std::stringstream ss;
+
+  ss << "_Unique_Name_" << unique_name_id++;
+  return ss.str();
+}
+
 void host_sleep(int* ptr)
 {
   int i = 0;
@@ -39,19 +54,18 @@ TEST(ResourceAwarePool_Host_Test, Check_States_Host)
   int* compare_ptr1 = ptr;
 
   EXPECT_EQ(getResource(pool, ptr), r1);
-  EXPECT_EQ(getPendingSize(pool), 0);
+  EXPECT_EQ(getNumPending(pool), 0);
 
   host_sleep(ptr);
 
   pool.deallocate(r1, ptr);
-  // EXPECT_EQ(getPendingSize(pool), 0); // When only using host, there will be no pending chunks
+  EXPECT_EQ(getNumPending(pool), 0); // When only using host, there will be no pending chunks
 
-  ptr = static_cast<int*>(pool.allocate(r2, 2048));
+  ptr = static_cast<int*>(pool.allocate(r2, 1024));
   int* compare_ptr2 = ptr;
 
   EXPECT_TRUE(r1 == r2);
   EXPECT_EQ(compare_ptr1, compare_ptr2); // only 1 host resource available, no possible data race
-
   pool.deallocate(r2, ptr);
 }
 
@@ -108,7 +122,7 @@ class ResourceAwarePoolTest : public ::testing::TestWithParam<std::string> {
   virtual void SetUp()
   {
     auto& rm = umpire::ResourceManager::getInstance();
-    m_pool = rm.makeAllocator<umpire::strategy::ResourceAwarePool>(std::string{"rap-pool-" + GetParam()},
+    m_pool = rm.makeAllocator<umpire::strategy::ResourceAwarePool>(std::string{"rap-pool-" + GetParam() + unique_name()},
                                                                    rm.getAllocator(GetParam()));
   }
 
@@ -122,35 +136,43 @@ class ResourceAwarePoolTest : public ::testing::TestWithParam<std::string> {
 
 TEST_P(ResourceAwarePoolTest, Check_States)
 {
-#if defined(UMPIRE_ENABLE_CUDA)
-  Cuda d1, d2;
-#elif defined(UMPIRE_ENABLE_HIP)
-  Hip d1, d2;
-#endif
-
+  resource_type d1, d2;
   Resource r1{d1}, r2{d2};
 
   double* ptr = static_cast<double*>(m_pool.allocate(r1, 1024));
 
   EXPECT_EQ(getResource(m_pool, ptr), r1);
-  EXPECT_EQ(getPendingSize(m_pool), 0);
+  EXPECT_EQ(getNumPending(m_pool), 0);
 
-#if defined(UMPIRE_ENABLE_CUDA)
   do_sleep<<<1, 32, 0, d1.get_stream()>>>(ptr);
-#elif defined(UMPIRE_ENABLE_HIP)
-  hipLaunchKernelGGL(do_sleep, 1, 32, 0, d1.get_stream(), ptr);
-#endif
 
   m_pool.deallocate(r1, ptr);
-  EXPECT_EQ(getPendingSize(m_pool), 1);
 
-  double* ptr2 = static_cast<double*>(m_pool.allocate(r2, 2048));
+  EXPECT_EQ(getNumPending(m_pool), 1);
+
+  double* ptr2 = static_cast<double*>(m_pool.allocate(r2, 1024));
 
   EXPECT_FALSE(r1 == r2);
   EXPECT_EQ(getResource(m_pool, ptr2), r2);
   EXPECT_NE(ptr, ptr2); // multiple device resources, possible data race, needs different addr
 }
 
+TEST_P(ResourceAwarePoolTest, ExplicitSync)
+{
+  resource_type d1, d2;
+
+  double* ptr = static_cast<double*>(m_pool.allocate(d1, 1024));
+
+  do_sleep<<<1, 32, 0, d1.get_stream()>>>(ptr);
+
+  m_pool.deallocate(d1, ptr);
+  d1.get_event().wait(); // explicitly sync the device streams (camp resources)
+  double* ptr2 = static_cast<double*>(m_pool.allocate(d2, 1024));
+
+  EXPECT_FALSE(d1 == d2);
+  EXPECT_EQ(ptr, ptr2); // multiple device resources, but with explicit sync, ptr is same
+}
+
 INSTANTIATE_TEST_SUITE_P(ResourceAwarePoolTests, ResourceAwarePoolTest, ::testing::ValuesIn(get_allocator_strings()));
 
 #endif