Skip to content

Commit

Permalink
Adding container config from #29
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattnappo
mattnappo committed May 31, 2023
1 parent eed6545 commit dc7da35
Show file tree
Hide file tree
Showing 5 changed files with 384 additions and 64 deletions.
49 changes: 49 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
# Prerequisites
*.d

# Compiled Object files
*.slo
*.lo
*.o
*.obj

# Precompiled Headers
*.gch
*.pch

# Compiled Dynamic libraries
*.so
*.dylib
*.dll

# Fortran module files
*.mod
*.smod

# Compiled Static libraries
*.lai
*.la
*.a
*.lib

# Executables
*.exe
*.out
*.app

# CMake
CMakeLists.txt.user
CMakeCache.txt
CMakeFiles
CMakeScripts
Testing
Makefile
cmake_install.cmake
install_manifest.txt
compile_commands.json
CTestTestfile.cmake
_deps
bin/
configuration/
volumes/
containers/config/htpasswd
63 changes: 60 additions & 3 deletions config/executor_manager.json
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,73 @@
{
"config": {
"rdma_device": "",
"rdma_device_port": 10000,
"rdma_device_port": 10004,
"resource_manager_address": "",
"resource_manager_port": 0,
"resource_manager_secret": 0
},
"executor": {
"use_docker": false,
"sandbox_type": "process",
"repetitions": 100,
"warmup_iters": 0,
"pin_threads": false
}
},
"sandbox-configuration": [
{
"key": "sarus",
"value": {
"index": 1,
"data": {
"user": "mcopik",
"name": "spcleth/hpc-disagg:rfaas-executor-daint",
"devices": [
"/dev/kgni0", "dev/kdreg"
],
"mounts": [
"/opt/cray",
"/tmp/drcc.sock",
"/etc/opt/cray/rdma-credentials",
"/etc/alternatives/cray-ugni",
"/etc/alternatives/cray-xpmem",
"/etc/alternatives/cray-alps",
"/etc/alternatives/cray-udreg",
"/etc/alternatives/cray-wlm_detect"
],
"mount_filesystem": [
"/scratch/snx3000/{user}"
],
"env": [
{
"key": "LD_LIBRARY_PATH",
"value": "/opt/cray/xpmem/default/lib64/;/opt/cray/udreg/default/lib64;/opt/cray/alps/default/lib64;/opt/cray/wlm_detect/default/lib64/"
}
]
}
}
},
{
"key": "docker",
"value": {
"index": 0,
"data": {
"image": "rfaas-registry/rfaas-base",
"network": "mynet",
"ip": "172.31.82.202",
"volume": "/home/ubuntu/rfaas/containers/opt",
"registry_ip": "172.31.82.200",
"registry_port": 5000
}
}
},
{
"key": "singularity",
"value": {
"index": 2,
"data": {
"container": "rfaas-container-singularity"
}
}
}
]
}

129 changes: 71 additions & 58 deletions server/executor_manager/executor_process.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@

#include <iterator>
#include <tuple>

#include <unistd.h>
Expand All @@ -8,6 +9,7 @@
#include <spdlog/spdlog.h>

#include <rdmalib/allocation.hpp>
#include <rdmalib/rdmalib.hpp>

#include "executor_process.hpp"
#include "settings.hpp"
Expand Down Expand Up @@ -81,27 +83,30 @@ namespace rfaas::executor_manager {
executor_pin_threads = std::to_string(0);//counter++);
else
executor_pin_threads = std::to_string(exec.pin_threads);
bool use_docker = exec.use_docker;
auto sandbox_type = exec.sandbox_type;

std::string mgr_port = std::to_string(conn.port);
std::string mgr_secret = std::to_string(conn.secret);
std::string mgr_buf_addr = std::to_string(conn.r_addr);
std::string mgr_buf_rkey = std::to_string(conn.r_key);

int mypid = fork();
int mypid = vfork();
if(mypid < 0) {
spdlog::error("Fork failed! {}", mypid);
}
if(mypid == 0) {
mypid = getpid();
auto out_file = ("executor_" + std::to_string(mypid));

spdlog::info("Child fork begins work on PID {}, using Docker? {}", mypid, use_docker);
spdlog::info("Child fork begins work on PID {}, using sandbox {}", mypid, sandbox_serialize(sandbox_type));
int fd = open(out_file.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
dup2(fd, 1);
dup2(fd, 2);
if(!use_docker) {
const char * argv[] = {

std::vector<std::string> argv;
std::vector<const char*> additional_args;
if(sandbox_type == SandboxType::PROCESS) {
argv = {
"executor",
"-a", client_addr.c_str(),
"-p", client_port.c_str(),
Expand All @@ -122,53 +127,43 @@ namespace rfaas::executor_manager {
"--mgr-buf-rkey", mgr_buf_rkey.c_str(),
nullptr
};
int ret = execvp(argv[0], const_cast<char**>(&argv[0]));
if(ret == -1) {
spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno));
close(fd);
exit(1);
}
} else {
//const char * argv[] = {
// "docker_rdma_sriov", "run",
// "--rm",
// "--net=mynet", "-i", //"-it",
// // FIXME: make configurable
// "--ip=148.187.105.220",
// // FIXME: make configurable
// "--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt",
// // FIXME: make configurable
// "rdma-test",
// "/opt/bin/executor",
// "-a", client_addr.c_str(),
// "-p", client_port.c_str(),
// "--polling-mgr", "thread",
// "-r", executor_repetitions.c_str(),
// "-x", executor_recv_buf.c_str(),
// "-s", client_in_size.c_str(),
// "--pin-threads", "true",
// "--fast", client_cores.c_str(),
// "--warmup-iters", executor_warmups.c_str(),
// "--max-inline-data", executor_max_inline.c_str(),
// "--func-size", client_func_size.c_str(),
// "--timeout", client_timeout.c_str(),
// "--mgr-address", conn.addr.c_str(),
// "--mgr-port", mgr_port.c_str(),
// "--mgr-secret", mgr_secret.c_str(),
// "--mgr-buf-addr", mgr_buf_addr.c_str(),
// "--mgr-buf-rkey", mgr_buf_rkey.c_str(),
// nullptr
//};
const char * argv[] = {
"docker_rdma_sriov", "run",
"--rm",
"--net=mynet", "-i", //"-it",
// FIXME: make configurable
"--ip=148.187.105.250",
// FIXME: make configurable
"--volume", "/users/mcopik/projects/rdma/repo/build_repo2:/opt",
// FIXME: make configurable
"rdma-test",
} else if (sandbox_type == SandboxType::SARUS) {
argv = {
"sarus", "run"
};

exec.sandbox_config->generate_args(argv, exec.sandbox_user);
argv.emplace_back(exec.sandbox_name);
argv.emplace_back(exec.sandbox_config->get_executor_path());

additional_args = {
"-a", client_addr.c_str(),
"-p", client_port.c_str(),
"--polling-mgr", "thread",
"-r", executor_repetitions.c_str(),
"-x", executor_recv_buf.c_str(),
"-s", client_in_size.c_str(),
"--pin-threads", executor_pin_threads.c_str(),
"--fast", client_cores.c_str(),
"--warmup-iters", executor_warmups.c_str(),
"--max-inline-data", executor_max_inline.c_str(),
"--func-size", client_func_size.c_str(),
"--timeout", client_timeout.c_str(),
"--mgr-address", conn.addr.c_str(),
"--mgr-port", mgr_port.c_str(),
"--mgr-secret", mgr_secret.c_str(),
"--mgr-buf-addr", mgr_buf_addr.c_str(),
"--mgr-buf-rkey", mgr_buf_rkey.c_str(),
nullptr
};
} else if (sandbox_type == SandboxType::DOCKER) {
argv = {
"docker_rdma_sriov", "run", "--rm", "-i"
};
DockerConfiguration config = std::get<DockerConfiguration>(*exec.sandbox_config);
config.generate_args(argv);

additional_args = {
"/opt/bin/executor",
"-a", client_addr.c_str(),
"-p", client_port.c_str(),
Expand All @@ -189,14 +184,32 @@ namespace rfaas::executor_manager {
"--mgr-buf-rkey", mgr_buf_rkey.c_str(),
nullptr
};
int ret = execvp(argv[0], const_cast<char**>(&argv[0]));
if(ret == -1) {
spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno));
close(fd);
exit(1);
}
} else if(sandbox_type == SandboxType::SINGULARITY) {
// Handle Singularity case
SingularityConfiguration config = std::get<SingularityConfiguration>(*exec.sandbox_config);
}

std::vector<const char*> cstrings_argv;
std::transform(argv.begin(), argv.end(), std::back_inserter(cstrings_argv),
[](const std::string & input) -> const char* {
return input.c_str();
}
);
std::copy(additional_args.begin(), additional_args.end(), std::back_inserter(cstrings_argv));

SPDLOG_DEBUG("Executor launch arguments");
for(const char* str : cstrings_argv)
if(str)
std::cout<<str<<std::endl;
//SPDLOG_DEBUG(str);

int ret = execvp(cstrings_argv.data()[0], const_cast<char**>(&cstrings_argv.data()[0]));
if(ret == -1) {
spdlog::error("Executor process failed {}, reason {}", errno, strerror(errno));
close(fd);
exit(1);
}

//close(fd);
exit(0);
}
Expand Down
Loading

0 comments on commit dc7da35

Please sign in to comment.