From 65d3e4e7fd7e15545f0dac1086c3138dba1fb00c Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 27 Sep 2024 20:26:35 +0200 Subject: [PATCH 1/7] [config] Update configuration file to the new standards --- config/benchmark.json | 2 +- config/executor_manager.json | 5 +++-- config/executors_database.json | 16 +++++++++------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config/benchmark.json b/config/benchmark.json index 6c438b1..681fa6f 100644 --- a/config/benchmark.json +++ b/config/benchmark.json @@ -6,6 +6,7 @@ "resource_manager_port": 0 }, "benchmark": { + "memory": 256, "pin_threads": false, "repetitions": 100, "warmup_repetitions": 0, @@ -13,4 +14,3 @@ "hot_timeout": -1 } } - diff --git a/config/executor_manager.json b/config/executor_manager.json index 6e4cf8b..b6ae953 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -2,9 +2,11 @@ "config": { "rdma_device": "", "rdma_device_port": 10000, + "node_name": "exec-mgr-node", "resource_manager_address": "", "resource_manager_port": 0, - "resource_manager_secret": 0 + "resource_manager_secret": 0, + "rdma-sleep": true }, "executor": { "use_docker": false, @@ -13,4 +15,3 @@ "pin_threads": false } } - diff --git a/config/executors_database.json b/config/executors_database.json index dba77c9..9aff0d0 100644 --- a/config/executors_database.json +++ b/config/executors_database.json @@ -1,9 +1,11 @@ { - "executors": [ - { - "address": "", - "port": 10000, - "cores": 0 - } - ] + "executors": [ + { + "node": "exec-mgr-node", + "address": "", + "port": 10000, + "cores": 1, + "memory": 512 + } + ] } From f7f0caebdfc765375f90f3a515d39be6cd650bea Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 27 Sep 2024 22:17:58 +0200 Subject: [PATCH 2/7] [rfaas] Support non-resource-manager path --- rfaas/include/rfaas/allocation.hpp | 4 +++ rfaas/include/rfaas/client.hpp | 2 +- rfaas/include/rfaas/executor.hpp | 3 ++- rfaas/lib/connection.cpp | 6 +++-- rfaas/lib/executor.cpp | 41 +++++++++++++++++++++++++++++- 5 files changed, 51 insertions(+), 5 deletions(-) diff --git a/rfaas/include/rfaas/allocation.hpp b/rfaas/include/rfaas/allocation.hpp index ed63705..af12eba 100644 --- a/rfaas/include/rfaas/allocation.hpp +++ b/rfaas/include/rfaas/allocation.hpp @@ -39,6 +39,10 @@ namespace rfaas { uint32_t func_buf_size; int32_t listen_port; char listen_address[16]; + + // Legacy support for skipping resource manager + int16_t cores = 0; + int32_t memory = 0; }; struct LeaseStatus { diff --git a/rfaas/include/rfaas/client.hpp b/rfaas/include/rfaas/client.hpp index 6388662..3a92ded 100644 --- a/rfaas/include/rfaas/client.hpp +++ b/rfaas/include/rfaas/client.hpp @@ -72,7 +72,7 @@ namespace rfaas { server_data instance = nodes_data.server(0); - return std::make_optional(instance.address, instance.port, cores, memory, -1, _device); + return std::make_optional(instance.address, instance.port, cores, memory, 0, _device); } private: diff --git a/rfaas/include/rfaas/executor.hpp b/rfaas/include/rfaas/executor.hpp index 17a857f..fb0e0e4 100644 --- a/rfaas/include/rfaas/executor.hpp +++ b/rfaas/include/rfaas/executor.hpp @@ -77,12 +77,13 @@ namespace rfaas { ~executor(); executor(executor&& obj); + executor& operator=(executor&& obj); bool connect(const std::string & ip, int port); // Skipping managers is useful for benchmarking bool allocate(std::string functions_path, int max_input_size, int hot_timeout, - bool skip_manager = false, rdmalib::Benchmarker<5> * benchmarker = nullptr); + bool skip_manager = false, bool skip_resource_manager = false, rdmalib::Benchmarker<5> * benchmarker = nullptr); void deallocate(); rdmalib::Buffer load_library(std::string path); void poll_queue(); diff --git a/rfaas/lib/connection.cpp b/rfaas/lib/connection.cpp index 7f866d4..9ed2511 100644 --- a/rfaas/lib/connection.cpp +++ b/rfaas/lib/connection.cpp @@ -122,13 +122,15 @@ namespace rfaas { int rcv_buf, int max_inline_data): _address(address), _port(port), - _active(_address, _port, rcv_buf), _rcv_buf_size(rcv_buf), _send_buffer(1), _receive_buffer(rcv_buf), _max_inline_data(max_inline_data) { - _active.allocate(); + if(!address.empty()) { + _active = rdmalib::RDMAActive(_address, _port, rcv_buf); + _active.allocate(); + } } bool resource_mgr_connection::connect() diff --git a/rfaas/lib/executor.cpp b/rfaas/lib/executor.cpp index a25d99f..faa52d0 100644 --- a/rfaas/lib/executor.cpp +++ b/rfaas/lib/executor.cpp @@ -92,6 +92,35 @@ namespace rfaas { obj._active_polling.store(false); } + executor& executor::operator=(executor&& obj) + { + + this->deallocate(); + + _state = std::move(obj._state); + _execs_buf = std::move(obj._execs_buf); + _device = std::move(obj._device); + _numcores = std::move(obj._numcores); + _memory = std::move(obj._memory); + _executions = std::move(obj._executions); + _invoc_id = std::move(obj._invoc_id); + _lease_id = std::move(obj._lease_id); + _connections = std::move(obj._connections); + _exec_manager = std::move(obj._exec_manager); + _func_names = std::move(obj._func_names); + _futures = std::move(obj._futures); + _background_thread = std::move(obj._background_thread); + + _end_requested = obj._end_requested.load(); + obj._end_requested.store(false); + + _active_polling = obj._active_polling.load(); + obj._active_polling.store(false); + + + return *this; + } + rdmalib::Buffer executor::load_library(std::string path) { _func_names.clear(); @@ -265,7 +294,7 @@ namespace rfaas { } bool executor::allocate(std::string functions_path, int max_input_size, - int hot_timeout, bool skip_manager, rdmalib::Benchmarker<5> * benchmarker) + int hot_timeout, bool skip_manager, bool skip_resource_manger, rdmalib::Benchmarker<5> * benchmarker) { rdmalib::Buffer functions = load_library(functions_path); @@ -275,6 +304,7 @@ namespace rfaas { if(benchmarker) benchmarker->start(); bool ret = _exec_manager->connect(); + spdlog::error("connect"); if(benchmarker) { benchmarker->end(0); benchmarker->start(); @@ -296,6 +326,15 @@ namespace rfaas { }; strcpy(_exec_manager->request().listen_address, _device.ip_address.c_str()); + // Legacy path + if(skip_resource_manger) { + + spdlog::error("{} {}", _numcores, _memory); + _exec_manager->request().cores = _numcores; + _exec_manager->request().memory = _memory; + + } + if(!_exec_manager->submit()) { return false; } From cdfdfc11dde52843e123f82ad7cd9776126422cf Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 27 Sep 2024 22:19:43 +0200 Subject: [PATCH 3/7] [exec-mgr] Support allocations done without resource mgr --- server/executor_manager/manager.cpp | 42 ++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/server/executor_manager/manager.cpp b/server/executor_manager/manager.cpp index 6c0310d..48e30b9 100644 --- a/server/executor_manager/manager.cpp +++ b/server/executor_manager/manager.cpp @@ -269,14 +269,26 @@ namespace rfaas::executor_manager { auto lease = _leases.get_threadsafe(lease_id); - if(!lease.has_value()) { - spdlog::warn("Received request for unknown lease {}", lease_id); - *_client_responses.data() = (LeaseStatus) {LeaseStatus::UNKNOWN}; - client.connection->post_send(_client_responses); - client.connection->receive_wcs().update_requests(-1); - client.connection->receive_wcs().refill(); - client.connection->poll_wc(rdmalib::QueueType::SEND, true, 1); - return true; + if(!_skip_rm) { + + if(!lease.has_value()) { + spdlog::warn("Received request for unknown lease {}", lease_id); + *_client_responses.data() = (LeaseStatus) {LeaseStatus::UNKNOWN}; + client.connection->post_send(_client_responses); + client.connection->receive_wcs().update_requests(-1); + client.connection->receive_wcs().refill(); + client.connection->poll_wc(rdmalib::QueueType::SEND, true, 1); + return true; + } + + } else { + + lease = Lease{ + client.allocation_requests.data()[wr_id].lease_id, + client.allocation_requests.data()[wr_id].cores, + client.allocation_requests.data()[wr_id].memory + }; + } rdmalib::PrivateData<0,0,32> data; @@ -498,12 +510,18 @@ namespace rfaas::executor_manager { client_poller.set_nonblocking(); client_poller.notify_events(false); - rdmalib::Poller res_mgr{_res_mgr_connection->connection().qp()->recv_cq}; - res_mgr.set_nonblocking(); - res_mgr.notify_events(false); + rdmalib::Poller res_mgr; + + if(!_skip_rm) { + + res_mgr = rdmalib::Poller{_res_mgr_connection->connection().qp()->recv_cq}; + res_mgr.set_nonblocking(); + res_mgr.notify_events(false); + event_poller.add_channel(res_mgr, 1); + + } event_poller.add_channel(client_poller, 0); - event_poller.add_channel(res_mgr, 1); std::vector poll_send; std::vector disconnections; From 01feea47778643dc54124d6480c4d36eb6e5c33f Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Fri, 27 Sep 2024 22:20:43 +0200 Subject: [PATCH 4/7] [benchmarks] Add again support for non resource mgr path in warm benchmarker --- benchmarks/warm_benchmark.cpp | 38 ++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/benchmarks/warm_benchmark.cpp b/benchmarks/warm_benchmark.cpp index 9e428d0..235ed89 100644 --- a/benchmarks/warm_benchmark.cpp +++ b/benchmarks/warm_benchmark.cpp @@ -41,21 +41,41 @@ int main(int argc, char **argv) { settings.resource_manager_address, settings.resource_manager_port, *settings.device ); - if (!instance.connect()) { - spdlog::error("Connection to resource manager failed!"); - return 1; - } - auto leased_executor = instance.lease(settings.benchmark.numcores, settings.benchmark.memory, *settings.device); - if (!leased_executor.has_value()) { - spdlog::error("Couldn't acquire a lease!"); - return 1; + bool skip_resource_manager = !opts.executors_database.empty(); + + std::optional leased_executor; + if (!skip_resource_manager) { + + if (!instance.connect()) { + spdlog::error("Connection to resource manager failed!"); + return 1; + } + + leased_executor = instance.lease(settings.benchmark.numcores, settings.benchmark.memory, *settings.device); + if (!leased_executor.has_value()) { + spdlog::error("Couldn't acquire a lease!"); + return 1; + } + + } else { + + std::ifstream in_cfg(opts.executors_database); + rfaas::servers::deserialize(in_cfg); + in_cfg.close(); + + leased_executor = instance.lease(rfaas::servers::instance(), settings.benchmark.numcores, settings.benchmark.memory); + if (!leased_executor.has_value()) { + spdlog::error("Couldn't acquire a lease!"); + return 1; + } + } rfaas::executor executor = std::move(leased_executor.value()); if (!executor.allocate(opts.flib, opts.input_size, - settings.benchmark.hot_timeout, false)) { + settings.benchmark.hot_timeout, false, skip_resource_manager)) { spdlog::error("Connection to executor and allocation failed!"); return 1; } From a84db29a891c52bd273a43d6b0dd3377cc808793 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Sat, 28 Sep 2024 02:21:11 +0200 Subject: [PATCH 5/7] [config] Further updates to default config files --- config/benchmark.json | 2 +- config/executor_manager.json | 4 ++-- config/executors_database.json | 2 +- config/resource_manager.json | 9 ++++++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config/benchmark.json b/config/benchmark.json index 681fa6f..8d0f213 100644 --- a/config/benchmark.json +++ b/config/benchmark.json @@ -1,7 +1,7 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10005, + "rdma_device_port": 10006, "resource_manager_address": "", "resource_manager_port": 0 }, diff --git a/config/executor_manager.json b/config/executor_manager.json index b6ae953..929810a 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -1,11 +1,11 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10000, + "rdma_device_port": 10005, "node_name": "exec-mgr-node", "resource_manager_address": "", "resource_manager_port": 0, - "resource_manager_secret": 0, + "resource_manager_secret": 42, "rdma-sleep": true }, "executor": { diff --git a/config/executors_database.json b/config/executors_database.json index 9aff0d0..ead8e75 100644 --- a/config/executors_database.json +++ b/config/executors_database.json @@ -3,7 +3,7 @@ { "node": "exec-mgr-node", "address": "", - "port": 10000, + "port": 10005, "cores": 1, "memory": 512 } diff --git a/config/resource_manager.json b/config/resource_manager.json index e735e02..da7461c 100644 --- a/config/resource_manager.json +++ b/config/resource_manager.json @@ -1,8 +1,11 @@ { "config": { "rdma_device": "", - "rdma_device_port": 0, - "http_network_address": "", - "http_network_port": 0 + "rdma_device_port": 10000, + "http_network_address": "0.0.0.0", + "http_network_port": 5000, + "rdma-threads": 1, + "rdma-secret": 42, + "rdma-sleep": true } } From c2439cf5cc7e9b4e461dfeb4a4fb97d4b4d378b9 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Sat, 28 Sep 2024 03:23:35 +0200 Subject: [PATCH 6/7] [res-mgr] Strange bug in the HTTP interface --- server/resource_manager/db.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/resource_manager/db.cpp b/server/resource_manager/db.cpp index c5129f4..a0e00c2 100644 --- a/server/resource_manager/db.cpp +++ b/server/resource_manager/db.cpp @@ -16,7 +16,7 @@ namespace rfaas { namespace resource_manager { ExecutorDB::ResultCode ExecutorDB::add(const std::string & node_name, const std::string & ip_address, int port, int cores, int memory) { - if(node_name.length() < rfaas::server_data::NODE_NAME_LENGTH) { + if(node_name.length() > rfaas::server_data::NODE_NAME_LENGTH) { return ResultCode::MALFORMED_DATA; } From 74054779498779bf5b0d25b97fcec7f2f6a635d4 Mon Sep 17 00:00:00 2001 From: Marcin Copik Date: Sat, 28 Sep 2024 03:23:42 +0200 Subject: [PATCH 7/7] [config] Update tutorial --- docs/tutorial.md | 147 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 21 deletions(-) diff --git a/docs/tutorial.md b/docs/tutorial.md index e1ec8b3..43febcb 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -9,10 +9,71 @@ global management of billing and it distributes data on active executor servers Here, we skip the deployment of resource manager for simplicity. On small deployments with just few executor servers, we can bypass this step. +### RDMA Configuration + +rFaaS currently supports InfiniBand and RoCE devices through the ibverbs library. +If you do not own such a device, you can still emulate it on a regular Ethernet network +with [SoftROCE](https://github.com/SoftRoCE). This kernel driver allows to create an emulated +RDMA device on top of a regular network. Of course, it won't be able to achieve the same performance +as a regular RDMA device but it will implement a similar set of functionalities. + +The installation of [SoftROCE] should be straightforward on most modern Linux distributions, +and it should not be necessary to manually compile and install kernel modules. + +For example, on Ubuntu-based distributions, you need to install `rdma-core`, `ibverbs-utils`. +Then, you can add a virtual RDMA device with the following command: + +``` +sudo rdma link add test type rxe netdev +``` + +where `netdev` is the name of your Ethernet device used for emulation. You can check the device +has been created with the following command: + +``` +ibv_devices + + device node GUID + ------ ---------------- + test 067bcbfffeb6f9f6 + +``` + +To fully check the configuration works, we can run a simple performance +test by using tools provided in the package: `perftest`. In one shell, open: + +``` +ib_write_bw +``` + +And in the second one, start: + +``` +ib_write_bw +``` + +Replace `` with the IP address of selected net device. +You should see the following output: + +``` +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] +Conflicting CPU frequency values detected: 3902.477000 != 400.000000. CPU Frequency is not max. + 65536 5000 419.30 361.10 0.005778 +--------------------------------------------------------------------------------------- +``` + +Another tool used that can be used is `rping` available in `rmadcm-utils` package: + +``` +rping -s -a -v + +rping -c -a -v +``` + ### Setup -We assume that rFaaS executor will be executed on one server, using the RDMA device `server_device` -and the network address `server_ip`. +We assume that rFaaS executor will be executed on one server, using the RDMA device `server_device` and the network address `server_ip`. Then, we're going to deploy the benchmarker on another machine, using the RDMA device `benchmark_device` and the netwrok address `benchmark_ip`. These four variables will be used when modifying JSON config files. @@ -47,12 +108,7 @@ We can use default values for maximal size of inline messaged and the receive bu } ``` -To defines these automatically, we can use a jq one-liner: - -``` -jq --arg device "$server_device" --arg address "$server_ip" '.devices[0].name = $device | .devices[0].ip_address = $address' /config/devices.json > server_devices.json -jq --arg device "$client_device" --arg address "$client_ip" '.devices[0].name = $device | .devices[0].ip_address = $address' /config/devices.json > client_devices.json -``` +To generate these automatically, we can the helper script `tools/device_generator.sh > devices.json` ### rFaaS function @@ -96,10 +152,12 @@ and it needs to be extended with device. { "config": { "rdma_device": "", - "rdma_device_port": , + "rdma_device_port": 10005, + "node_name": "exec-mgr-node", "resource_manager_address": "", "resource_manager_port": 0, - "resource_manager_secret": 0 + "resource_manager_secret": 42, + "rdma-sleep": true }, "executor": { "use_docker": false, @@ -129,8 +187,8 @@ After starting the manager, you should see the output similar to this: ```console [13:12:31:629452] [P 425702] [T 425702] [info] Executing rFaaS executor manager! -[13:12:31:634632] [P 425702] [T 425702] [info] Listening on device rocep61s0, port 10006 -[13:12:31:634674] [P 425702] [T 425702] [info] Begin listening at 192.168.0.21:10006 and processing events! +[13:12:31:634632] [P 425702] [T 425702] [info] Listening on device rocep61s0, port 10005 +[13:12:31:634674] [P 425702] [T 425702] [info] Begin listening at 192.168.0.21:10005 and processing events! ``` ### Benchmark Example @@ -144,13 +202,15 @@ configuration from the previous step: ```json { - "executors": [ - { - "port": 10000, - "cores": 1, - "address": "" - } - ] + "executors": [ + { + "node": "exec-mgr-node", + "address": "" + "port": 10005, + "cores": 1, + "memory": 512 + } + ] } ``` @@ -172,7 +232,7 @@ value describes the hot polling timeout in milliseconds. { "config": { "rdma_device": "", - "rdma_device_port": 10005, + "rdma_device_port": 10006, "resource_manager_address": "", "resource_manager_port": 0 }, @@ -189,7 +249,7 @@ value describes the hot polling timeout in milliseconds. We generate the configuration using the following command: ``` -jq --arg device "$client_device" '.config.rdma_device = $device' ../repo2/config/benchmark.json > benchmark.json +jq --arg device "$client_device" '.config.rdma_device = $device' /config/benchmark.json > benchmark.json ``` To start a benchmark instance with the `name` functions from `examples/libfunctions.so`, @@ -214,3 +274,48 @@ Data: 1 For details about this and other benchmarks, please take a look [at the documentation](benchmarks.md). +### Using Resource Manager + +For large deployments, we deploy the resource manager to control leases. + +An example of a configuration is available in `config/resource_manager.json` +and it needs to be extended with device selection. +By default, we assume that resource manager uses port 10000 for RDMA connections, +and port 5000 for the HTTP server. + +```json +{ + "config": { + "rdma_device": "", + "rdma_device_port": 10000, + "http_network_address": "0.0.0.0", + "http_network_port": 5000, + "rdma-threads": 1, + "rdma-secret": 42, + "rdma-sleep": true + } +} +``` + +We can use the following command to generate the configuration: + +``` +jq --arg device "$server_device" '.config.rdma_device = $device' /config/resource_manager.json > resource_manager.json +``` + +To start an instance of the resource manager, we use the following command: + +``` +bin/resource_manager -c config/resource_manager.json --device-database server_devices.json -i executors_database.json +``` + +Here, we populate the database of all executors by providing a JSON list generated previously: `-i executors_database.json`. +Alternatively, we can use the HTTP interface designed for integration with batch managers, and send a POST request that adds a new executor: + +``` +curl http://127.0.0.1:5000/add\?node\=exec-mgr-node -X POST -d '{"ip_address": "192.168.0.29", "port": 10005, "cores": 1, "memory": 512}' +``` + +Then, we only need to modify the configuration of `executor_manager.json` and `benchmark.json` to add the IP address and port of resource manager. +For executor manager, we remove the `--skip-resource-manager` flag. For benchmarker, we remove the `--executors-database executors_database.json` +parameter, as all executor data will now be handled by the resource manager.