diff --git a/benchmarks/warm_benchmark.cpp b/benchmarks/warm_benchmark.cpp index 9e428d0..235ed89 100644 --- a/benchmarks/warm_benchmark.cpp +++ b/benchmarks/warm_benchmark.cpp @@ -41,21 +41,41 @@ int main(int argc, char **argv) { settings.resource_manager_address, settings.resource_manager_port, *settings.device ); - if (!instance.connect()) { - spdlog::error("Connection to resource manager failed!"); - return 1; - } - auto leased_executor = instance.lease(settings.benchmark.numcores, settings.benchmark.memory, *settings.device); - if (!leased_executor.has_value()) { - spdlog::error("Couldn't acquire a lease!"); - return 1; + bool skip_resource_manager = !opts.executors_database.empty(); + + std::optional leased_executor; + if (!skip_resource_manager) { + + if (!instance.connect()) { + spdlog::error("Connection to resource manager failed!"); + return 1; + } + + leased_executor = instance.lease(settings.benchmark.numcores, settings.benchmark.memory, *settings.device); + if (!leased_executor.has_value()) { + spdlog::error("Couldn't acquire a lease!"); + return 1; + } + + } else { + + std::ifstream in_cfg(opts.executors_database); + rfaas::servers::deserialize(in_cfg); + in_cfg.close(); + + leased_executor = instance.lease(rfaas::servers::instance(), settings.benchmark.numcores, settings.benchmark.memory); + if (!leased_executor.has_value()) { + spdlog::error("Couldn't acquire a lease!"); + return 1; + } + } rfaas::executor executor = std::move(leased_executor.value()); if (!executor.allocate(opts.flib, opts.input_size, - settings.benchmark.hot_timeout, false)) { + settings.benchmark.hot_timeout, false, skip_resource_manager)) { spdlog::error("Connection to executor and allocation failed!"); return 1; } diff --git a/config/benchmark.json b/config/benchmark.json index 6c438b1..8d0f213 100644 --- a/config/benchmark.json +++ b/config/benchmark.json @@ -1,11 +1,12 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10005, + "rdma_device_port": 10006, "resource_manager_address": "", "resource_manager_port": 0 }, "benchmark": { + "memory": 256, "pin_threads": false, "repetitions": 100, "warmup_repetitions": 0, @@ -13,4 +14,3 @@ "hot_timeout": -1 } } - diff --git a/config/executor_manager.json b/config/executor_manager.json index 6e4cf8b..929810a 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -1,10 +1,12 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10000, + "rdma_device_port": 10005, + "node_name": "exec-mgr-node", "resource_manager_address": "", "resource_manager_port": 0, - "resource_manager_secret": 0 + "resource_manager_secret": 42, + "rdma-sleep": true }, "executor": { "use_docker": false, @@ -13,4 +15,3 @@ "pin_threads": false } } - diff --git a/config/executors_database.json b/config/executors_database.json index dba77c9..ead8e75 100644 --- a/config/executors_database.json +++ b/config/executors_database.json @@ -1,9 +1,11 @@ { - "executors": [ - { - "address": "", - "port": 10000, - "cores": 0 - } - ] + "executors": [ + { + "node": "exec-mgr-node", + "address": "", + "port": 10005, + "cores": 1, + "memory": 512 + } + ] } diff --git a/config/resource_manager.json b/config/resource_manager.json index e735e02..da7461c 100644 --- a/config/resource_manager.json +++ b/config/resource_manager.json @@ -1,8 +1,11 @@ { "config": { "rdma_device": "", - "rdma_device_port": 0, - "http_network_address": "", - "http_network_port": 0 + "rdma_device_port": 10000, + "http_network_address": "0.0.0.0", + "http_network_port": 5000, + "rdma-threads": 1, + "rdma-secret": 42, + "rdma-sleep": true } } diff --git a/docs/tutorial.md b/docs/tutorial.md index e1ec8b3..43febcb 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -9,10 +9,71 @@ global management of billing and it distributes data on active executor servers Here, we skip the deployment of resource manager for simplicity. On small deployments with just few executor servers, we can bypass this step. +### RDMA Configuration + +rFaaS currently supports InfiniBand and RoCE devices through the ibverbs library. +If you do not own such a device, you can still emulate it on a regular Ethernet network +with [SoftROCE](https://github.com/SoftRoCE). This kernel driver allows to create an emulated +RDMA device on top of a regular network. Of course, it won't be able to achieve the same performance +as a regular RDMA device but it will implement a similar set of functionalities. + +The installation of [SoftROCE] should be straightforward on most modern Linux distributions, +and it should not be necessary to manually compile and install kernel modules. + +For example, on Ubuntu-based distributions, you need to install `rdma-core`, `ibverbs-utils`. +Then, you can add a virtual RDMA device with the following command: + +``` +sudo rdma link add test type rxe netdev +``` + +where `netdev` is the name of your Ethernet device used for emulation. You can check the device +has been created with the following command: + +``` +ibv_devices + + device node GUID + ------ ---------------- + test 067bcbfffeb6f9f6 + +``` + +To fully check the configuration works, we can run a simple performance +test by using tools provided in the package: `perftest`. In one shell, open: + +``` +ib_write_bw +``` + +And in the second one, start: + +``` +ib_write_bw +``` + +Replace `` with the IP address of selected net device. +You should see the following output: + +``` +--------------------------------------------------------------------------------------- + #bytes #iterations BW peak[MB/sec] BW average[MB/sec] MsgRate[Mpps] +Conflicting CPU frequency values detected: 3902.477000 != 400.000000. CPU Frequency is not max. + 65536 5000 419.30 361.10 0.005778 +--------------------------------------------------------------------------------------- +``` + +Another tool used that can be used is `rping` available in `rmadcm-utils` package: + +``` +rping -s -a -v + +rping -c -a -v +``` + ### Setup -We assume that rFaaS executor will be executed on one server, using the RDMA device `server_device` -and the network address `server_ip`. +We assume that rFaaS executor will be executed on one server, using the RDMA device `server_device` and the network address `server_ip`. Then, we're going to deploy the benchmarker on another machine, using the RDMA device `benchmark_device` and the netwrok address `benchmark_ip`. These four variables will be used when modifying JSON config files. @@ -47,12 +108,7 @@ We can use default values for maximal size of inline messaged and the receive bu } ``` -To defines these automatically, we can use a jq one-liner: - -``` -jq --arg device "$server_device" --arg address "$server_ip" '.devices[0].name = $device | .devices[0].ip_address = $address' /config/devices.json > server_devices.json -jq --arg device "$client_device" --arg address "$client_ip" '.devices[0].name = $device | .devices[0].ip_address = $address' /config/devices.json > client_devices.json -``` +To generate these automatically, we can the helper script `tools/device_generator.sh > devices.json` ### rFaaS function @@ -96,10 +152,12 @@ and it needs to be extended with device. { "config": { "rdma_device": "", - "rdma_device_port": , + "rdma_device_port": 10005, + "node_name": "exec-mgr-node", "resource_manager_address": "", "resource_manager_port": 0, - "resource_manager_secret": 0 + "resource_manager_secret": 42, + "rdma-sleep": true }, "executor": { "use_docker": false, @@ -129,8 +187,8 @@ After starting the manager, you should see the output similar to this: ```console [13:12:31:629452] [P 425702] [T 425702] [info] Executing rFaaS executor manager! -[13:12:31:634632] [P 425702] [T 425702] [info] Listening on device rocep61s0, port 10006 -[13:12:31:634674] [P 425702] [T 425702] [info] Begin listening at 192.168.0.21:10006 and processing events! +[13:12:31:634632] [P 425702] [T 425702] [info] Listening on device rocep61s0, port 10005 +[13:12:31:634674] [P 425702] [T 425702] [info] Begin listening at 192.168.0.21:10005 and processing events! ``` ### Benchmark Example @@ -144,13 +202,15 @@ configuration from the previous step: ```json { - "executors": [ - { - "port": 10000, - "cores": 1, - "address": "" - } - ] + "executors": [ + { + "node": "exec-mgr-node", + "address": "" + "port": 10005, + "cores": 1, + "memory": 512 + } + ] } ``` @@ -172,7 +232,7 @@ value describes the hot polling timeout in milliseconds. { "config": { "rdma_device": "", - "rdma_device_port": 10005, + "rdma_device_port": 10006, "resource_manager_address": "", "resource_manager_port": 0 }, @@ -189,7 +249,7 @@ value describes the hot polling timeout in milliseconds. We generate the configuration using the following command: ``` -jq --arg device "$client_device" '.config.rdma_device = $device' ../repo2/config/benchmark.json > benchmark.json +jq --arg device "$client_device" '.config.rdma_device = $device' /config/benchmark.json > benchmark.json ``` To start a benchmark instance with the `name` functions from `examples/libfunctions.so`, @@ -214,3 +274,48 @@ Data: 1 For details about this and other benchmarks, please take a look [at the documentation](benchmarks.md). +### Using Resource Manager + +For large deployments, we deploy the resource manager to control leases. + +An example of a configuration is available in `config/resource_manager.json` +and it needs to be extended with device selection. +By default, we assume that resource manager uses port 10000 for RDMA connections, +and port 5000 for the HTTP server. + +```json +{ + "config": { + "rdma_device": "", + "rdma_device_port": 10000, + "http_network_address": "0.0.0.0", + "http_network_port": 5000, + "rdma-threads": 1, + "rdma-secret": 42, + "rdma-sleep": true + } +} +``` + +We can use the following command to generate the configuration: + +``` +jq --arg device "$server_device" '.config.rdma_device = $device' /config/resource_manager.json > resource_manager.json +``` + +To start an instance of the resource manager, we use the following command: + +``` +bin/resource_manager -c config/resource_manager.json --device-database server_devices.json -i executors_database.json +``` + +Here, we populate the database of all executors by providing a JSON list generated previously: `-i executors_database.json`. +Alternatively, we can use the HTTP interface designed for integration with batch managers, and send a POST request that adds a new executor: + +``` +curl http://127.0.0.1:5000/add\?node\=exec-mgr-node -X POST -d '{"ip_address": "192.168.0.29", "port": 10005, "cores": 1, "memory": 512}' +``` + +Then, we only need to modify the configuration of `executor_manager.json` and `benchmark.json` to add the IP address and port of resource manager. +For executor manager, we remove the `--skip-resource-manager` flag. For benchmarker, we remove the `--executors-database executors_database.json` +parameter, as all executor data will now be handled by the resource manager. diff --git a/rfaas/include/rfaas/allocation.hpp b/rfaas/include/rfaas/allocation.hpp index ed63705..af12eba 100644 --- a/rfaas/include/rfaas/allocation.hpp +++ b/rfaas/include/rfaas/allocation.hpp @@ -39,6 +39,10 @@ namespace rfaas { uint32_t func_buf_size; int32_t listen_port; char listen_address[16]; + + // Legacy support for skipping resource manager + int16_t cores = 0; + int32_t memory = 0; }; struct LeaseStatus { diff --git a/rfaas/include/rfaas/client.hpp b/rfaas/include/rfaas/client.hpp index 6388662..3a92ded 100644 --- a/rfaas/include/rfaas/client.hpp +++ b/rfaas/include/rfaas/client.hpp @@ -72,7 +72,7 @@ namespace rfaas { server_data instance = nodes_data.server(0); - return std::make_optional(instance.address, instance.port, cores, memory, -1, _device); + return std::make_optional(instance.address, instance.port, cores, memory, 0, _device); } private: diff --git a/rfaas/include/rfaas/executor.hpp b/rfaas/include/rfaas/executor.hpp index 17a857f..fb0e0e4 100644 --- a/rfaas/include/rfaas/executor.hpp +++ b/rfaas/include/rfaas/executor.hpp @@ -77,12 +77,13 @@ namespace rfaas { ~executor(); executor(executor&& obj); + executor& operator=(executor&& obj); bool connect(const std::string & ip, int port); // Skipping managers is useful for benchmarking bool allocate(std::string functions_path, int max_input_size, int hot_timeout, - bool skip_manager = false, rdmalib::Benchmarker<5> * benchmarker = nullptr); + bool skip_manager = false, bool skip_resource_manager = false, rdmalib::Benchmarker<5> * benchmarker = nullptr); void deallocate(); rdmalib::Buffer load_library(std::string path); void poll_queue(); diff --git a/rfaas/lib/connection.cpp b/rfaas/lib/connection.cpp index 7f866d4..9ed2511 100644 --- a/rfaas/lib/connection.cpp +++ b/rfaas/lib/connection.cpp @@ -122,13 +122,15 @@ namespace rfaas { int rcv_buf, int max_inline_data): _address(address), _port(port), - _active(_address, _port, rcv_buf), _rcv_buf_size(rcv_buf), _send_buffer(1), _receive_buffer(rcv_buf), _max_inline_data(max_inline_data) { - _active.allocate(); + if(!address.empty()) { + _active = rdmalib::RDMAActive(_address, _port, rcv_buf); + _active.allocate(); + } } bool resource_mgr_connection::connect() diff --git a/rfaas/lib/executor.cpp b/rfaas/lib/executor.cpp index a25d99f..faa52d0 100644 --- a/rfaas/lib/executor.cpp +++ b/rfaas/lib/executor.cpp @@ -92,6 +92,35 @@ namespace rfaas { obj._active_polling.store(false); } + executor& executor::operator=(executor&& obj) + { + + this->deallocate(); + + _state = std::move(obj._state); + _execs_buf = std::move(obj._execs_buf); + _device = std::move(obj._device); + _numcores = std::move(obj._numcores); + _memory = std::move(obj._memory); + _executions = std::move(obj._executions); + _invoc_id = std::move(obj._invoc_id); + _lease_id = std::move(obj._lease_id); + _connections = std::move(obj._connections); + _exec_manager = std::move(obj._exec_manager); + _func_names = std::move(obj._func_names); + _futures = std::move(obj._futures); + _background_thread = std::move(obj._background_thread); + + _end_requested = obj._end_requested.load(); + obj._end_requested.store(false); + + _active_polling = obj._active_polling.load(); + obj._active_polling.store(false); + + + return *this; + } + rdmalib::Buffer executor::load_library(std::string path) { _func_names.clear(); @@ -265,7 +294,7 @@ namespace rfaas { } bool executor::allocate(std::string functions_path, int max_input_size, - int hot_timeout, bool skip_manager, rdmalib::Benchmarker<5> * benchmarker) + int hot_timeout, bool skip_manager, bool skip_resource_manger, rdmalib::Benchmarker<5> * benchmarker) { rdmalib::Buffer functions = load_library(functions_path); @@ -275,6 +304,7 @@ namespace rfaas { if(benchmarker) benchmarker->start(); bool ret = _exec_manager->connect(); + spdlog::error("connect"); if(benchmarker) { benchmarker->end(0); benchmarker->start(); @@ -296,6 +326,15 @@ namespace rfaas { }; strcpy(_exec_manager->request().listen_address, _device.ip_address.c_str()); + // Legacy path + if(skip_resource_manger) { + + spdlog::error("{} {}", _numcores, _memory); + _exec_manager->request().cores = _numcores; + _exec_manager->request().memory = _memory; + + } + if(!_exec_manager->submit()) { return false; } diff --git a/server/executor_manager/manager.cpp b/server/executor_manager/manager.cpp index 6c0310d..48e30b9 100644 --- a/server/executor_manager/manager.cpp +++ b/server/executor_manager/manager.cpp @@ -269,14 +269,26 @@ namespace rfaas::executor_manager { auto lease = _leases.get_threadsafe(lease_id); - if(!lease.has_value()) { - spdlog::warn("Received request for unknown lease {}", lease_id); - *_client_responses.data() = (LeaseStatus) {LeaseStatus::UNKNOWN}; - client.connection->post_send(_client_responses); - client.connection->receive_wcs().update_requests(-1); - client.connection->receive_wcs().refill(); - client.connection->poll_wc(rdmalib::QueueType::SEND, true, 1); - return true; + if(!_skip_rm) { + + if(!lease.has_value()) { + spdlog::warn("Received request for unknown lease {}", lease_id); + *_client_responses.data() = (LeaseStatus) {LeaseStatus::UNKNOWN}; + client.connection->post_send(_client_responses); + client.connection->receive_wcs().update_requests(-1); + client.connection->receive_wcs().refill(); + client.connection->poll_wc(rdmalib::QueueType::SEND, true, 1); + return true; + } + + } else { + + lease = Lease{ + client.allocation_requests.data()[wr_id].lease_id, + client.allocation_requests.data()[wr_id].cores, + client.allocation_requests.data()[wr_id].memory + }; + } rdmalib::PrivateData<0,0,32> data; @@ -498,12 +510,18 @@ namespace rfaas::executor_manager { client_poller.set_nonblocking(); client_poller.notify_events(false); - rdmalib::Poller res_mgr{_res_mgr_connection->connection().qp()->recv_cq}; - res_mgr.set_nonblocking(); - res_mgr.notify_events(false); + rdmalib::Poller res_mgr; + + if(!_skip_rm) { + + res_mgr = rdmalib::Poller{_res_mgr_connection->connection().qp()->recv_cq}; + res_mgr.set_nonblocking(); + res_mgr.notify_events(false); + event_poller.add_channel(res_mgr, 1); + + } event_poller.add_channel(client_poller, 0); - event_poller.add_channel(res_mgr, 1); std::vector poll_send; std::vector disconnections; diff --git a/server/resource_manager/db.cpp b/server/resource_manager/db.cpp index c5129f4..a0e00c2 100644 --- a/server/resource_manager/db.cpp +++ b/server/resource_manager/db.cpp @@ -16,7 +16,7 @@ namespace rfaas { namespace resource_manager { ExecutorDB::ResultCode ExecutorDB::add(const std::string & node_name, const std::string & ip_address, int port, int cores, int memory) { - if(node_name.length() < rfaas::server_data::NODE_NAME_LENGTH) { + if(node_name.length() > rfaas::server_data::NODE_NAME_LENGTH) { return ResultCode::MALFORMED_DATA; }