diff --git a/benchmarks/warm_benchmark.cpp b/benchmarks/warm_benchmark.cpp index d16f36b..427cef2 100644 --- a/benchmarks/warm_benchmark.cpp +++ b/benchmarks/warm_benchmark.cpp @@ -13,6 +13,7 @@ #include #include +#include #include "warm_benchmark.hpp" #include "settings.hpp" @@ -68,19 +69,56 @@ int main(int argc, char ** argv) } // FIXME: move me to a memory allocator - rdmalib::Buffer in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size); - in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE); - out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); - memset(in.data(), 0, opts.input_size); - for(int i = 0; i < opts.input_size; ++i) { - ((char*)in.data())[i] = 1; + + // Sample: test demonstrating standard memory allocation. + + // rdmalib::Buffer in(opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE), out(opts.input_size); + // in.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE); + // out.register_memory(executor._state.pd(), IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE); + + + // Sample: test demonstrating allocation with our custom allocator. + + // rfaas::RdmaInfo info_in(executor,IBV_ACCESS_LOCAL_WRITE,rdmalib::functions::Submission::DATA_HEADER_SIZE); + // rfaas::RdmaAllocator> allocator_in{info_in}; + // rdmalib::Buffer* in0 = allocator_in.allocate(opts.input_size); + // allocator_in.construct(in0, opts.input_size, rdmalib::functions::Submission::DATA_HEADER_SIZE); + // + // rfaas::RdmaInfo info_out(executor,(IBV_ACCESS_LOCAL_WRITE| IBV_ACCESS_REMOTE_WRITE)); + // rfaas::RdmaAllocator> allocator_out{info_out}; + // rdmalib::Buffer* out0 = allocator_out.allocate(opts.input_size); + // allocator_out.construct(out0, opts.input_size); + + + // Sample: test demonstrating allocation with std::vector. + + rfaas::RdmaInfo info_v_in(executor, IBV_ACCESS_LOCAL_WRITE, rdmalib::functions::Submission::DATA_HEADER_SIZE); + rfaas::RdmaAllocator> allocator_v_in{info_v_in}; + std::vector, rfaas::RdmaAllocator>> v_in(allocator_v_in); + + rfaas::RdmaInfo info_v_out(executor, (IBV_ACCESS_LOCAL_WRITE | IBV_ACCESS_REMOTE_WRITE)); + rfaas::RdmaAllocator> allocator_v_out{info_v_out}; + std::vector, rfaas::RdmaAllocator>> v_out(allocator_v_out); + + v_in.emplace_back(static_cast(opts.input_size), rdmalib::functions::Submission::DATA_HEADER_SIZE); + v_out.emplace_back(static_cast(opts.input_size)); + + rdmalib::Buffer *in = &v_in[0]; + rdmalib::Buffer *out = &v_out[0]; + + + // TODO: Since the for loop writes a value of 1 to each byte of the in buffer, + // it overwrites all bytes previously set to 0 by the memset() function. + memset(in->data(), 0, opts.input_size); + for (int i = 0; i < opts.input_size; ++i) { + ((char *) in->data())[i] = 1; } rdmalib::Benchmarker<1> benchmarker{settings.benchmark.repetitions}; spdlog::info("Warmups begin"); for(int i = 0; i < settings.benchmark.warmup_repetitions; ++i) { SPDLOG_DEBUG("Submit warm {}", i); - executor.execute(opts.fname, in, out); + executor.execute(opts.fname, *in, *out); } spdlog::info("Warmups completed"); @@ -88,7 +126,7 @@ int main(int argc, char ** argv) for(int i = 0; i < settings.benchmark.repetitions;) { benchmarker.start(); SPDLOG_DEBUG("Submit execution {}", i); - auto ret = executor.execute(opts.fname, in, out); + auto ret = executor.execute(opts.fname, *in, *out); if(std::get<0>(ret)) { SPDLOG_DEBUG("Finished execution {} out of {}", i, settings.benchmark.repetitions); benchmarker.end(0); @@ -108,8 +146,12 @@ int main(int argc, char ** argv) printf("Data: "); for(int i = 0; i < std::min(100, opts.input_size); ++i) - printf("%d ", ((char*)out.data())[i]); + printf("%d ", ((char*)out->data())[i]); printf("\n"); +// std::free(&v_in); +// std::free(&v_out); +// v_in.get_allocator().deallocate(&v_in[0],opts.input_size); +// v_out.get_allocator().deallocate(&v_out[0],opts.input_size); return 0; } diff --git a/rfaas/include/rfaas/rdma_allocator.hpp b/rfaas/include/rfaas/rdma_allocator.hpp new file mode 100644 index 0000000..731db11 --- /dev/null +++ b/rfaas/include/rfaas/rdma_allocator.hpp @@ -0,0 +1,81 @@ +// +// Created by mou on 4/2/23. +// + +#ifndef __RFAAS_RDMA_ALLOCATOR_HPP__ +#define __RFAAS_RDMA_ALLOCATOR_HPP__ + +#include +#include +#include +#include + +namespace rfaas { + + struct RdmaInfo { + + public: + RdmaInfo(executor &executor, const int &access, const int &header_size = 0) + : executor(executor), access(access), header_size(header_size) {} + + const executor &executor; + const int &access; + const int &header_size = 0; + }; + + template + class RdmaAllocator { + + public: + typedef T value_type; + + inline constexpr explicit RdmaAllocator(RdmaInfo &info) noexcept: _info(info) {} + + template + inline constexpr explicit RdmaAllocator(const RdmaAllocator &) noexcept {} + + [[nodiscard]] inline T *allocate(const size_t &size) { + if (size > std::numeric_limits::max() / sizeof(T)) + throw std::bad_array_new_length(); + + if (auto p = static_cast(std::malloc(size * sizeof(T) + _info.header_size))) { + report(p, size * sizeof(T) + _info.header_size); + return p; + } + throw std::bad_alloc(); + } + + template + inline void construct(U *p, Args &&... args) { + ::new(p) U(std::forward(args)...); + p->register_memory(_info.executor._state.pd(), _info.access); + } + + inline void deallocate(T *p, std::size_t size) noexcept { + report(p, size, 0); + std::free(p); + } + + template + struct rebind { + using other = RdmaAllocator; + }; + + private: + const RdmaInfo &_info; + + inline void report(T *p, std::size_t size, bool alloc = true) const { + std::cout << (alloc ? "Alloc: " : "Dealloc: ") << size + << " bytes at " << std::hex << std::showbase + << reinterpret_cast(p) << std::dec << '\n'; + } + }; + + template + inline bool operator==(const RdmaAllocator &, const RdmaAllocator &) { return true; } + + template + inline bool operator!=(const RdmaAllocator &, const RdmaAllocator &) { return false; } +} + +#endif //__RFAAS_RDMA_ALLOCATOR_HPP__ \ No newline at end of file