From dc7da3532f02bab93741530cd51eeb8c6654db05 Mon Sep 17 00:00:00 2001 From: Matt Nappo Date: Wed, 31 May 2023 16:40:45 +0000 Subject: [PATCH] Adding container config from #29 --- .gitignore | 49 +++++++ config/executor_manager.json | 63 ++++++++- server/executor_manager/executor_process.cpp | 129 ++++++++++--------- server/executor_manager/settings.cpp | 89 +++++++++++++ server/executor_manager/settings.hpp | 118 ++++++++++++++++- 5 files changed, 384 insertions(+), 64 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8364c9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,49 @@ +# Prerequisites +*.d + +# Compiled Object files +*.slo +*.lo +*.o +*.obj + +# Precompiled Headers +*.gch +*.pch + +# Compiled Dynamic libraries +*.so +*.dylib +*.dll + +# Fortran module files +*.mod +*.smod + +# Compiled Static libraries +*.lai +*.la +*.a +*.lib + +# Executables +*.exe +*.out +*.app + +# CMake +CMakeLists.txt.user +CMakeCache.txt +CMakeFiles +CMakeScripts +Testing +Makefile +cmake_install.cmake +install_manifest.txt +compile_commands.json +CTestTestfile.cmake +_deps +bin/ +configuration/ +volumes/ +containers/config/htpasswd diff --git a/config/executor_manager.json b/config/executor_manager.json index 6e4cf8b..4695330 100644 --- a/config/executor_manager.json +++ b/config/executor_manager.json @@ -1,16 +1,73 @@ { "config": { "rdma_device": "", - "rdma_device_port": 10000, + "rdma_device_port": 10004, "resource_manager_address": "", "resource_manager_port": 0, "resource_manager_secret": 0 }, "executor": { - "use_docker": false, + "sandbox_type": "process", "repetitions": 100, "warmup_iters": 0, "pin_threads": false - } + }, + "sandbox-configuration": [ + { + "key": "sarus", + "value": { + "index": 1, + "data": { + "user": "mcopik", + "name": "spcleth/hpc-disagg:rfaas-executor-daint", + "devices": [ + "/dev/kgni0", "dev/kdreg" + ], + "mounts": [ + "/opt/cray", + "/tmp/drcc.sock", + "/etc/opt/cray/rdma-credentials", + "/etc/alternatives/cray-ugni", + "/etc/alternatives/cray-xpmem", + "/etc/alternatives/cray-alps", + "/etc/alternatives/cray-udreg", + "/etc/alternatives/cray-wlm_detect" + ], + "mount_filesystem": [ + "/scratch/snx3000/{user}" + ], + "env": [ + { + "key": "LD_LIBRARY_PATH", + "value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/" + } + ] + } + } + }, + { + "key": "docker", + "value": { + "index": 0, + "data": { + "image": "rfaas-registry/rfaas-base", + "network": "mynet", + "ip": "172.31.82.202", + "volume": "/home/ubuntu/rfaas/containers/opt", + "registry_ip": "172.31.82.200", + "registry_port": 5000 + } + } + }, + { + "key": "singularity", + "value": { + "index": 2, + "data": { + "container": "rfaas-container-singularity" + } + } + } + ] } diff --git a/server/executor_manager/executor_process.cpp b/server/executor_manager/executor_process.cpp index afdbf2d..f7b5afa 100644 --- a/server/executor_manager/executor_process.cpp +++ b/server/executor_manager/executor_process.cpp @@ -1,4 +1,5 @@ +#include #include #include @@ -8,6 +9,7 @@ #include #include +#include #include "executor_process.hpp" #include "settings.hpp" @@ -81,14 +83,14 @@ namespace rfaas::executor_manager { executor_pin_threads = std::to_string(0);//counter++); else executor_pin_threads = std::to_string(exec.pin_threads); - bool use_docker = exec.use_docker; + auto sandbox_type = exec.sandbox_type; std::string mgr_port = std::to_string(conn.port); std::string mgr_secret = std::to_string(conn.secret); std::string mgr_buf_addr = std::to_string(conn.r_addr); std::string mgr_buf_rkey = std::to_string(conn.r_key); - int mypid = fork(); + int mypid = vfork(); if(mypid < 0) { spdlog::error("Fork failed! {}", mypid); } @@ -96,12 +98,15 @@ namespace rfaas::executor_manager { mypid = getpid(); auto out_file = ("executor_" + std::to_string(mypid)); - spdlog::info("Child fork begins work on PID {}, using Docker? {}", mypid, use_docker); + spdlog::info("Child fork begins work on PID {}, using sandbox {}", mypid, sandbox_serialize(sandbox_type)); int fd = open(out_file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); dup2(fd, 1); dup2(fd, 2); - if(!use_docker) { - const char * argv[] = { + + std::vector argv; + std::vector additional_args; + if(sandbox_type == SandboxType::PROCESS) { + argv = { "executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), @@ -122,53 +127,43 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - int ret = execvp(argv[0], const_cast(&argv[0])); - if(ret == -1) { - spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); - close(fd); - exit(1); - } - } else { - //const char * argv[] = { - // "docker_rdma_sriov", "run", - // "--rm", - // "--net=mynet", "-i", //"-it", - // // FIXME: make configurable - // "--ip=148.187.105.220", - // // FIXME: make configurable - // "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // // FIXME: make configurable - // "rdma-test", - // "/opt/bin/executor", - // "-a", client_addr.c_str(), - // "-p", client_port.c_str(), - // "--polling-mgr", "thread", - // "-r", executor_repetitions.c_str(), - // "-x", executor_recv_buf.c_str(), - // "-s", client_in_size.c_str(), - // "--pin-threads", "true", - // "--fast", client_cores.c_str(), - // "--warmup-iters", executor_warmups.c_str(), - // "--max-inline-data", executor_max_inline.c_str(), - // "--func-size", client_func_size.c_str(), - // "--timeout", client_timeout.c_str(), - // "--mgr-address", conn.addr.c_str(), - // "--mgr-port", mgr_port.c_str(), - // "--mgr-secret", mgr_secret.c_str(), - // "--mgr-buf-addr", mgr_buf_addr.c_str(), - // "--mgr-buf-rkey", mgr_buf_rkey.c_str(), - // nullptr - //}; - const char * argv[] = { - "docker_rdma_sriov", "run", - "--rm", - "--net=mynet", "-i", //"-it", - // FIXME: make configurable - "--ip=148.187.105.250", - // FIXME: make configurable - "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt", - // FIXME: make configurable - "rdma-test", + } else if (sandbox_type == SandboxType::SARUS) { + argv = { + "sarus", "run" + }; + + exec.sandbox_config->generate_args(argv, exec.sandbox_user); + argv.emplace_back(exec.sandbox_name); + argv.emplace_back(exec.sandbox_config->get_executor_path()); + + additional_args = { + "-a", client_addr.c_str(), + "-p", client_port.c_str(), + "--polling-mgr", "thread", + "-r", executor_repetitions.c_str(), + "-x", executor_recv_buf.c_str(), + "-s", client_in_size.c_str(), + "--pin-threads", executor_pin_threads.c_str(), + "--fast", client_cores.c_str(), + "--warmup-iters", executor_warmups.c_str(), + "--max-inline-data", executor_max_inline.c_str(), + "--func-size", client_func_size.c_str(), + "--timeout", client_timeout.c_str(), + "--mgr-address", conn.addr.c_str(), + "--mgr-port", mgr_port.c_str(), + "--mgr-secret", mgr_secret.c_str(), + "--mgr-buf-addr", mgr_buf_addr.c_str(), + "--mgr-buf-rkey", mgr_buf_rkey.c_str(), + nullptr + }; + } else if (sandbox_type == SandboxType::DOCKER) { + argv = { + "docker_rdma_sriov", "run", "--rm", "-i" + }; + DockerConfiguration config = std::get(*exec.sandbox_config); + config.generate_args(argv); + + additional_args = { "/opt/bin/executor", "-a", client_addr.c_str(), "-p", client_port.c_str(), @@ -189,14 +184,32 @@ namespace rfaas::executor_manager { "--mgr-buf-rkey", mgr_buf_rkey.c_str(), nullptr }; - int ret = execvp(argv[0], const_cast(&argv[0])); - if(ret == -1) { - spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); - close(fd); - exit(1); - } + } else if(sandbox_type == SandboxType::SINGULARITY) { + // Handle Singularity case + SingularityConfiguration config = std::get(*exec.sandbox_config); + } + std::vector cstrings_argv; + std::transform(argv.begin(), argv.end(), std::back_inserter(cstrings_argv), + [](const std::string & input) -> const char* { + return input.c_str(); + } + ); + std::copy(additional_args.begin(), additional_args.end(), std::back_inserter(cstrings_argv)); + + SPDLOG_DEBUG("Executor launch arguments"); + for(const char* str : cstrings_argv) + if(str) + std::cout<(&cstrings_argv.data()[0])); + if(ret == -1) { + spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno)); + close(fd); + exit(1); } + //close(fd); exit(0); } diff --git a/server/executor_manager/settings.cpp b/server/executor_manager/settings.cpp index a5fcc14..b87364b 100644 --- a/server/executor_manager/settings.cpp +++ b/server/executor_manager/settings.cpp @@ -1,16 +1,44 @@ +#include +#include + #include +#include + #include "settings.hpp" namespace rfaas::executor_manager { + SandboxType sandbox_deserialize(std::string type) + { + static std::map sandboxes = { + {"process", SandboxType::PROCESS}, + {"docker", SandboxType::DOCKER}, + {"sarus", SandboxType::SARUS}, + {"singularity", SandboxType::SINGULARITY} + }; + return sandboxes.at(type); + } + + std::string sandbox_serialize(SandboxType type) + { + static std::map sandboxes = { + {SandboxType::PROCESS, "process"}, + {SandboxType::DOCKER, "docker"}, + {SandboxType::SARUS, "sarus"}, + {SandboxType::SINGULARITY, "singularity"} + }; + return sandboxes.at(type); + } + Settings Settings::deserialize(std::istream & in) { Settings settings{}; cereal::JSONInputArchive archive_in(in); archive_in(cereal::make_nvp("config", settings)); archive_in(cereal::make_nvp("executor", settings.exec)); + archive_in(cereal::make_nvp("sandbox-configuration", settings.sandboxes)); // read RDMA device details rfaas::device_data * dev = rfaas::devices::instance().device(settings.rdma_device); @@ -24,8 +52,69 @@ namespace rfaas::executor_manager { settings.exec.max_inline_data = dev->max_inline_data; settings.exec.recv_buffer_size = dev->default_receive_buffer_size; + if (settings.exec.sandbox_type != SandboxType::PROCESS) { + settings.exec.sandbox_config = &settings.sandboxes.at(settings.exec.sandbox_type); + } + // FIXME: should be sent with request + if (settings.exec.sandbox_type == SandboxType::SARUS) { + SarusConfiguration config = std::get(*settings.exec.sandbox_config); + settings.exec.sandbox_user = config.user; + settings.exec.sandbox_name = config.name; + for(auto & mount : config.mounts) + std::cerr << mount << std::endl; + } + return settings; } + void SarusConfiguration::generate_args(std::vector & args, const std::string & user) const + { + for(auto & dev : this->devices) + args.emplace_back(rdmalib::impl::string_format("--device=%s", dev)); + + for(auto & mount : this->mount_filesystem) { + std::string user_partition{mount}; + user_partition = std::regex_replace(user_partition, std::regex{R"(\{user\})"}, user); + args.emplace_back(rdmalib::impl::string_format("--mount=type=bind,source=%s,destination=%s", user_partition, user_partition)); + } + + for(auto & mount : this->mounts) + args.emplace_back(rdmalib::impl::string_format("--mount=type=bind,source=%s,destination=%s", mount, mount)); + + for(auto & [key, value] : this->env) { + args.emplace_back("-e"); + args.emplace_back(rdmalib::impl::string_format("%s=%s", key, value)); + } + + } + + std::string SarusConfiguration::get_executor_path() const + { + // Horrible hack - we need to get the location of the executor. + // We assume that rFaaS is built on the shared filesystem that is mounted + // in the container. + // Furtheermore, since rFaaS is built in a single directory, we know + // that executor should be located in the same directory as + // executor_manager. + + // This works only on Linux! + auto path = std::filesystem::canonical("/proc/self/exe").parent_path(); + return path / "executor"; + } + + void DockerConfiguration::generate_args(std::vector & args) const { + std::string ip_arg = "--ip=" + ip; + std::string volume_arg = volume + ":/opt"; + std::string net_arg = "--net=" + network; + std::string registry_port = std::to_string(this->registry_port); + std::string docker_image = registry_ip + ":" + registry_port + "/" + image; + + args.emplace_back(net_arg); + args.emplace_back(ip_arg); + args.emplace_back("--volume"); + args.emplace_back(volume_arg); + args.emplace_back(docker_image); + } } + diff --git a/server/executor_manager/settings.hpp b/server/executor_manager/settings.hpp index 5347c07..5f0a61e 100644 --- a/server/executor_manager/settings.hpp +++ b/server/executor_manager/settings.hpp @@ -1,19 +1,109 @@ - #ifndef __RFAAS_EXECUTOR_MANAGER_SETTINGS_HPP__ #define __RFAAS_EXECUTOR_MANAGER_SETTINGS_HPP__ +#include #include #include #include +#include +#include +#include +#include + +namespace rfaas::executor_manager { + + enum class SandboxType { + PROCESS = 0, + DOCKER = 1, + SARUS = 2, + SINGULARITY = 3, + }; + + SandboxType sandbox_deserialize(std::string type); + + std::string sandbox_serialize(SandboxType type); + +} namespace rfaas::executor_manager { + struct DockerConfiguration { + std::string image; + std::string network; + std::string ip; + std::string volume; + std::string registry_ip; + int registry_port; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(image), CEREAL_NVP(network), + CEREAL_NVP(ip), CEREAL_NVP(volume), + CEREAL_NVP(registry_ip), CEREAL_NVP(registry_port) + ); + } + + void generate_args(std::vector & args) const; + }; + + struct SarusConfiguration { + std::string user; + std::string name; + std::vector devices; + std::vector mounts; + std::vector mount_filesystem; + std::map env; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(user), CEREAL_NVP(name), + CEREAL_NVP(devices), CEREAL_NVP(mounts), + CEREAL_NVP(mount_filesystem), CEREAL_NVP(env) + ); + } + + void generate_args(std::vector & args, const std::string & user) const; + + /** + * In the Sarus container, we cannot build the executor since we need to + * compile with Cray headers. + * Thus, we have to mount the rFaaS build directory and access executor + * this way. + **/ + std::string get_executor_path() const; + + }; + + struct SingularityConfiguration { + // TODO: Add other singularity options here + std::string container; + + template + void load(Archive & ar) + { + ar( + CEREAL_NVP(container) + ); + } + }; + + using SandboxConfiguration = std::variant; + struct ExecutorSettings { - bool use_docker; + SandboxType sandbox_type; + SandboxConfiguration* sandbox_config; + std::string sandbox_user; + std::string sandbox_name; + int repetitions; int warmup_iters; int recv_buffer_size; @@ -23,10 +113,12 @@ namespace rfaas::executor_manager { template void load(Archive & ar ) { + std::string sandbox_type; ar( - CEREAL_NVP(use_docker), CEREAL_NVP(repetitions), + CEREAL_NVP(sandbox_type), CEREAL_NVP(repetitions), CEREAL_NVP(warmup_iters), CEREAL_NVP(pin_threads) ); + this->sandbox_type = sandbox_deserialize(sandbox_type); } }; @@ -46,6 +138,8 @@ namespace rfaas::executor_manager { // Passed to the scheduled executor ExecutorSettings exec; + std::map sandboxes; + template void load(Archive & ar ) { @@ -61,4 +155,22 @@ namespace rfaas::executor_manager { } +namespace cereal +{ + + template inline + std::string save_minimal(Archive const &, rfaas::executor_manager::SandboxType const & t) + { + return rfaas::executor_manager::sandbox_serialize(t); + } + + template inline + void load_minimal( Archive const &, rfaas::executor_manager::SandboxType & t, std::string const & value) + { + t = rfaas::executor_manager::sandbox_deserialize(value); + } + +} + #endif +