[Resolve #18] Add C++ allocator

benchmarks/warm_benchmark.cpp

@@ -13,6 +13,7 @@
 
 #include <rfaas/executor.hpp>
 #include <rfaas/resources.hpp>
+#include <rfaas/rdma_allocator.hpp>
 
 #include "warm_benchmark.hpp"
 #include "settings.hpp"
@@ -68,27 +69,64 @@ int main(int argc, char ** argv)
   }
 
   // FIXME: move me to a memory allocator
-  rdmalib::Buffer<char> in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size);
-  in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE);
-  out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
-  memset(in.data(), 0, opts.input_size);
-  for(int i = 0; i < opts.input_size; ++i) {
-    ((char*)in.data())[i] = 1;
+
+  // Sample: test demonstrating standard memory allocation.
+
+  // rdmalib::Buffer<char> in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size);
+  // in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE);
+  // out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE);
+
+
+  // Sample: test demonstrating allocation with our custom allocator.
+
+  //  rfaas::RdmaInfo info_in(executor,IBV_ACCESS_LOCAL_WRITE,rdmalib::functions::Submission::DATA_HEADER_SIZE);
+  //  rfaas::RdmaAllocator<rdmalib::Buffer<char>> allocator_in{info_in};
+  //  rdmalib::Buffer<char>* in0 = allocator_in.allocate(opts.input_size);
+  //  allocator_in.construct(in0, opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE);
+  //
+  //  rfaas::RdmaInfo info_out(executor,(IBV_ACCESS_LOCAL_WRITE| IBV_ACCESS_REMOTE_WRITE));
+  //  rfaas::RdmaAllocator<rdmalib::Buffer<char>> allocator_out{info_out};
+  //  rdmalib::Buffer<char>* out0 = allocator_out.allocate(opts.input_size);
+  //  allocator_out.construct(out0, opts.input_size);
+
+
+  // Sample: test demonstrating allocation with std::vector.
+
+  rfaas::RdmaInfo info_v_in(executor, IBV_ACCESS_LOCAL_WRITE, rdmalib::functions::Submission::DATA_HEADER_SIZE);
+  rfaas::RdmaAllocator<rdmalib::Buffer<char>> allocator_v_in{info_v_in};
+  std::vector<rdmalib::Buffer<char>, rfaas::RdmaAllocator<rdmalib::Buffer<char>>> v_in(allocator_v_in);
+
+  rfaas::RdmaInfo info_v_out(executor, (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE));
+  rfaas::RdmaAllocator<rdmalib::Buffer<char>> allocator_v_out{info_v_out};
+  std::vector<rdmalib::Buffer<char>, rfaas::RdmaAllocator<rdmalib::Buffer<char>>> v_out(allocator_v_out);
+
+  v_in.emplace_back(static_cast<size_t>(opts.input_size), rdmalib::functions::Submission::DATA_HEADER_SIZE);
+  v_out.emplace_back(static_cast<size_t>(opts.input_size));
+
+  rdmalib::Buffer<char> *in = &v_in[0];
+  rdmalib::Buffer<char> *out = &v_out[0];
+
+
+  // TODO: Since the for loop writes a value of 1 to each byte of the in buffer,
+  //       it overwrites all bytes previously set to 0 by the memset() function.
+  memset(in->data(), 0, opts.input_size);
+  for (int i = 0; i < opts.input_size; ++i) {
+    ((char *) in->data())[i] = 1;
   }
 
   rdmalib::Benchmarker<1> benchmarker{settings.benchmark.repetitions};
   spdlog::info("Warmups begin");
   for(int i = 0; i < settings.benchmark.warmup_repetitions; ++i) {
     SPDLOG_DEBUG("Submit warm {}", i);
-    executor.execute(opts.fname, in, out);
+    executor.execute(opts.fname, *in, *out);
   }
   spdlog::info("Warmups completed");
 
   // Start actual measurements
   for(int i = 0; i < settings.benchmark.repetitions;) {
     benchmarker.start();
     SPDLOG_DEBUG("Submit execution {}", i);
-    auto ret = executor.execute(opts.fname, in, out);
+    auto ret = executor.execute(opts.fname, *in, *out);
     if(std::get<0>(ret)) {
       SPDLOG_DEBUG("Finished execution {} out of {}", i, settings.benchmark.repetitions);
       benchmarker.end(0);
@@ -108,8 +146,12 @@ int main(int argc, char ** argv)
 
   printf("Data: ");
   for(int i = 0; i < std::min(100, opts.input_size); ++i)
-    printf("%d ", ((char*)out.data())[i]);
+    printf("%d ", ((char*)out->data())[i]);
   printf("\n");
 
+//  std::free(&v_in);
+//  std::free(&v_out);
+//  v_in.get_allocator().deallocate(&v_in[0],opts.input_size);
+//  v_out.get_allocator().deallocate(&v_out[0],opts.input_size);
   return 0;
 }

rfaas/include/rfaas/rdma_allocator.hpp

@@ -0,0 +1,81 @@
+//
+// Created by mou on 4/2/23.
+//
+
+#ifndef __RFAAS_RDMA_ALLOCATOR_HPP__
+#define __RFAAS_RDMA_ALLOCATOR_HPP__
+
+#include <sys/mman.h>
+#include <cstddef>
+#include <rdmalib/buffer.hpp>
+#include <rfaas/executor.hpp>
+
+namespace rfaas {
+
+  struct RdmaInfo {
+
+  public:
+    RdmaInfo(executor &executor, const int &access, const int &header_size = 0)
+        : executor(executor), access(access), header_size(header_size) {}
+
+    const executor &executor;
+    const int &access;
+    const int &header_size = 0;
+  };
+
+  template<typename T>
+  class RdmaAllocator {
+
+  public:
+    typedef T value_type;
+
+    inline constexpr explicit RdmaAllocator(RdmaInfo &info) noexcept: _info(info) {}
+
+    template<class U>
+    inline constexpr explicit RdmaAllocator(const RdmaAllocator<U> &) noexcept {}
+
+    [[nodiscard]] inline T *allocate(const size_t &size) {
+      if (size > std::numeric_limits<std::size_t>::max() / sizeof(T))
+        throw std::bad_array_new_length();
+
+      if (auto p = static_cast<T *>(std::malloc(size * sizeof(T) + _info.header_size))) {
+        report(p, size * sizeof(T) + _info.header_size);
+        return p;
+      }
+      throw std::bad_alloc();
+    }
+
+    template<typename U, typename... Args>
+    inline void construct(U *p, Args &&... args) {
+      ::new(p) U(std::forward<Args>(args)...);
+      p->register_memory(_info.executor._state.pd(), _info.access);
+    }
+
+    inline void deallocate(T *p, std::size_t size) noexcept {
+      report(p, size, 0);
+      std::free(p);
+    }
+
+    template<typename U>
+    struct rebind {
+      using other = RdmaAllocator<U>;
+    };
+
+  private:
+    const RdmaInfo &_info;
+
+    inline void report(T *p, std::size_t size, bool alloc = true) const {
+      std::cout << (alloc ? "Alloc: " : "Dealloc: ") << size
+                << " bytes at " << std::hex << std::showbase
+                << reinterpret_cast<void *>(p) << std::dec << '\n';
+    }
+  };
+
+  template<class T, class U>
+  inline bool operator==(const RdmaAllocator<T> &, const RdmaAllocator<U> &) { return true; }
+
+  template<class T, class U>
+  inline bool operator!=(const RdmaAllocator<T> &, const RdmaAllocator<U> &) { return false; }
+}
+
+#endif //__RFAAS_RDMA_ALLOCATOR_HPP__