From 361aa1361d8b9c930d8e126fb0a19a641026fa1c Mon Sep 17 00:00:00 2001 From: JackyWoo Date: Fri, 17 May 2024 17:32:08 +0800 Subject: [PATCH] Remove useless tests --- src/Service/tests/CMakeLists.txt | 7 - src/Service/tests/raft_service_client.cpp | 196 ------------ src/Service/tests/raft_tcp_client.cpp | 47 --- src/Service/tests/raft_unit_benchmark.cpp | 345 ---------------------- 4 files changed, 595 deletions(-) delete mode 100644 src/Service/tests/raft_service_client.cpp delete mode 100644 src/Service/tests/raft_tcp_client.cpp delete mode 100644 src/Service/tests/raft_unit_benchmark.cpp diff --git a/src/Service/tests/CMakeLists.txt b/src/Service/tests/CMakeLists.txt index 645e8b7f12a..e69de29bb2d 100644 --- a/src/Service/tests/CMakeLists.txt +++ b/src/Service/tests/CMakeLists.txt @@ -1,7 +0,0 @@ -add_executable(raft_tcp_client raft_tcp_client.cpp) -add_executable(raft_service_client raft_service_client.cpp) -add_executable(raft_unit_benchmark raft_unit_benchmark.cpp raft_test_common.cpp) - -target_link_libraries(raft_tcp_client PRIVATE rk_zookeeper) -target_link_libraries(raft_service_client PRIVATE rk_zookeeper loggers) -target_link_libraries(raft_unit_benchmark PRIVATE nuraft rk_common_io rk_zookeeper loggers) diff --git a/src/Service/tests/raft_service_client.cpp b/src/Service/tests/raft_service_client.cpp deleted file mode 100644 index b956deac754..00000000000 --- a/src/Service/tests/raft_service_client.cpp +++ /dev/null @@ -1,196 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace Coordination; -using namespace RK; - -namespace RK -{ -class TestServer : public Poco::Util::Application, public Loggers -{ -public: - TestServer() = default; - ~TestServer() override = default; - void init(int argc, char ** argv) - { - char * log_level = argv[7]; - - /// Don't parse options with Poco library, we prefer neat boost::program_options - stopOptionsProcessing(); - /// Save received data into the internal config. - config().setBool("stacktrace", true); - config().setBool("logger.console", true); - config().setString("logger.log", "./benchmark_test.logs"); - config().setString("logger.level", log_level); - config().setBool("ignore-error", false); - - std::vector arguments; - for (int arg_num = 1; arg_num < argc; ++arg_num) - arguments.emplace_back(argv[arg_num]); - argsToConfig(arguments, config(), 100); - - if (config().has("logger.console") || config().has("logger.level") || config().has("logger.log")) - { - // force enable logging - config().setString("logger", "logger"); - // sensitive data rules are not used here - buildLoggers(config(), logger(), "clickhouse-local"); - } - } -}; - -} - -void getCurrentTime(String & date_str) -{ - const char time_fmt[] = "%Y%m%d%H%M%S"; - time_t curr_time; - time(&curr_time); - char tmp_buf[24]; - std::strftime(tmp_buf, sizeof(tmp_buf), time_fmt, localtime(&curr_time)); - date_str = tmp_buf; -} - -//IP PORT THREAD_SIZE SEND_COUNT KEY_SIZE VALUE_SIZE -int main(int argc, char ** argv) -{ - if (argc < 8) - { - std::cout << "Please run:xxx ip port thread_size send_count key_size value_size debug/trace" << std::endl; - return 0; - } - char buf[64]; - char * ip = argv[1]; - char * port_str = argv[2]; - snprintf(buf, 64, "%s:%s", ip, port_str); - String ip_port(buf); - std::cout << "Connect server[" << ip_port << "]." << std::endl; - - int thread_size = 5; - int send_count = 1000000; - int key_size = 256; - int value_size = 1024; - - thread_size = atoi(argv[3]); - send_count = atoi(argv[4]); - key_size = atoi(argv[5]); - value_size = atoi(argv[6]); - - std::cout << "thread_size " << thread_size << ", send_count " << send_count << ", key_size " << key_size << ", value_size " - << value_size << std::endl; - - RK::TestServer app; - app.init(argc, argv); - app.run(); - - auto * log = &Poco::Logger::get("raft_service_client"); - LOG_INFO(log, "Run test server!"); - - String identity; - std::vector hosts_strings; - hosts_strings.emplace_back(ip_port); - Coordination::ZooKeeper::Nodes nodes; - nodes.reserve(hosts_strings.size()); - - for (auto & host_string : hosts_strings) - { - try - { - bool secure = startsWith(host_string, "secure://"); - if (secure) - host_string.erase(0, strlen("secure://")); - - nodes.emplace_back(Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{host_string}, secure}); - } - catch (const Poco::Net::DNSException & e) - { - LOG_ERROR(log, "Cannot use SvsKeeper host {}, reason: {}", host_string, e.displayText()); - } - } - - String key = "/"; - //8+6=14 byte - String curr_time; - getCurrentTime(curr_time); - key.append(curr_time); - for (int i = 0; i < key_size - 25; i++) - { - key.append("k"); - } - - LOG_INFO(log, "Create prefix key {}", key); - - //1024 byte - String data; - for (int i = 0; i < value_size; i++) - { - data.append("v"); - } - - sleep(1); - - FreeThreadPool thread_pool(thread_size); - Stopwatch watch; - watch.start(); - for (int i = 0; i < thread_size; i++) - { - thread_pool.scheduleOrThrowOnError([ip_port, i, thread_size, send_count, &key, &data] { - auto * thread_log = &Poco::Logger::get("client_thread"); - char key_buf[257]; - zkutil::ZooKeeper::Ptr zookeeper = std::make_shared(ip_port, "", 60000, 30000); - int log_count = send_count / thread_size; - int begin = i * log_count; - int end = (i + 1) * log_count; - LOG_INFO(thread_log, "Begin run thread {}/{}, send_count {}, range[{} - {}) ", i, thread_size, send_count, begin, end); - while (begin < end) - { - snprintf(key_buf, 257, "%s%010d", key.data(), begin); - Coordination::Error ret = Coordination::Error::ZOK; - try - { - ret = zookeeper->tryCreate(key_buf, data, zkutil::CreateMode::Persistent); - } - catch (RK::Exception & ex) - { - LOG_INFO( - thread_log, "Response code {}, errmsg {}, key {}, exception {}", ret, errorMessage(ret), key_buf, ex.message()); - sleep(1); - //make new - zookeeper = std::make_shared(ip_port, "", 60000, 30000); - } - begin++; - } - }); - } - LOG_INFO(log, "Max thread count {}, running {}", thread_pool.getMaxThreads(), thread_pool.active()); - thread_pool.wait(); - watch.stop(); - int mill_second = watch.elapsedMilliseconds(); - int log_size = ((key_size + value_size) + sizeof(UInt32) * 4 + sizeof(UInt32) * 6); - double total_size = 1.0 * log_size * send_count / 1000 / 1000; - double byte_rate = 1.0 * total_size / mill_second * 1000; - double count_rate = 1.0 * send_count / mill_second * 1000; - LOG_INFO( - log, - "Append performance, size {}, count {}, total_size {}, micro second {}, byte rate {}, count rate {}", - log_size, - send_count, - total_size, - mill_second, - byte_rate, - count_rate); - return 0; -} diff --git a/src/Service/tests/raft_tcp_client.cpp b/src/Service/tests/raft_tcp_client.cpp deleted file mode 100644 index db2db3f2c8b..00000000000 --- a/src/Service/tests/raft_tcp_client.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace Coordination; - - -int main(int, char **) -{ - String identity; - - std::vector hosts_strings; - hosts_strings.emplace_back("127.0.0.1:8101"); - Coordination::ZooKeeper::Nodes nodes; - nodes.reserve(hosts_strings.size()); - auto * log = &Poco::Logger::get("raft_tcp_client"); - for (auto & host_string : hosts_strings) - { - try - { - bool secure = (startsWith(host_string, "secure://")); - - if (secure) - host_string.erase(0, strlen("secure://")); - - nodes.emplace_back(Coordination::ZooKeeper::Node{Poco::Net::SocketAddress{host_string}, secure}); - } - catch (const Poco::Net::DNSException & e) - { - LOG_ERROR(log, "Cannot use SvsKeeper host {}, reason: {}", host_string, e.displayText()); - } - } - - zkutil::ZooKeeper::Ptr zookeeper - = std::make_shared(hosts_strings[0], identity.empty() ? "" : "digest", 60000, 30000); - - for (size_t i = 0; i < 100; ++i) - { - zookeeper->asyncCreate("/testdfg" + std::to_string(i), "", zkutil::CreateMode::Persistent); - } - -} diff --git a/src/Service/tests/raft_unit_benchmark.cpp b/src/Service/tests/raft_unit_benchmark.cpp deleted file mode 100644 index fe5416d48d3..00000000000 --- a/src/Service/tests/raft_unit_benchmark.cpp +++ /dev/null @@ -1,345 +0,0 @@ -#include -#include -#include -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include - -using namespace Coordination; -using namespace RK; -using namespace nuraft; - -namespace RK -{ - -int parseLine(char * line) -{ - // This assumes that a digit will be found and the line ends in " Kb". - int i = strlen(line); - const char * p = line; - while (*p < '0' || *p > '9') - p++; - line[i - 3] = '\0'; - i = atoi(p); - return i; -} - -struct ProcessMem -{ - uint32_t virtual_mem; - uint32_t physical_mem; -}; - -ProcessMem getProcessMem() -{ - FILE * file = fopen("/proc/self/status", "r"); - char line[128]; - ProcessMem process_mem; - - while (fgets(line, 128, file) != nullptr) - { - if (strncmp(line, "VmSize:", 7) == 0) - { - process_mem.virtual_mem = parseLine(line); - break; - } - - if (strncmp(line, "VmRSS:", 6) == 0) - { - process_mem.physical_mem = parseLine(line); - break; - } - } - fclose(file); - return process_mem; -} - -} - -void getCurrentTime(String & date_str) -{ - const char time_fmt[] = "%Y%m%d%H%M%S"; - time_t curr_time; - time(&curr_time); - char tmp_buf[24]; - std::strftime(tmp_buf, sizeof(tmp_buf), time_fmt, localtime(&curr_time)); - date_str = tmp_buf; -} - -#define ASSERT_EQ_LOG(log, v1, v2) \ - { \ - if (v1 != v2) \ - LOG_WARNING(log, "v1 {}, v2 {} not equal.", v1, v2); \ - } - -void logSegmentThread() -{ - Poco::Logger * log = &(Poco::Logger::get("RaftLog")); - String log_dir(LOG_DIR + "/10"); - cleanDirectory(log_dir); - - auto log_store = LogSegmentStore::getInstance(log_dir, true); - //10M - UInt32 max_seg_count = 10; - ASSERT_EQ_LOG(log, log_store->init(10000000, max_seg_count), 0) - - int key_bytes = 256; - int value_bytes = 1024; - //256 byte - String key; - for (int i = 0; i < key_bytes; i++) - { - key.append("k"); - } - //1024 byte - String data; - for (int i = 0; i < value_bytes; i++) - { - data.append("v"); - } - - //std::mutex index_mutex; - std::vector thread_vec = {16}; - std::atomic log_index = 0; - int log_count = 100000; - for (auto thread_count : thread_vec) - { - //int end_index = log_index + log_count; - int thread_log_count = log_count / thread_count; - FreeThreadPool thread_pool(thread_count); - Stopwatch watch; - watch.start(); - for (int thread_idx = 0; thread_idx < thread_count; thread_idx++) - { - thread_pool.trySchedule( - [&log_store, &log_index, thread_count, thread_idx, thread_log_count, log_count, max_seg_count, &key, &data] - { - UInt64 log_idx; - UInt64 term = 1; - - auto * thread_log = &Poco::Logger::get("client_thread"); - LOG_INFO( - thread_log, - "Begin run thread size {}/{}, append count {}/{}", - thread_idx, - thread_count, - thread_log_count, - log_count); - - for (auto idx = 0; idx < thread_log_count; idx++) - { - auto entry_log = createLogEntry(term, key, data); - try - { - log_idx = log_store->appendEntry(entry_log); - } - catch (std::exception & ex) - { - LOG_ERROR(thread_log, "append exception : {}", ex.what()); - continue; - } - - ptr new_log; - try - { - new_log = log_store->getEntry(log_idx); - } - catch (std::exception & ex) - { - LOG_ERROR(thread_log, "get exception : {}", ex.what()); - continue; - } - if (new_log == nullptr) - { - continue; - } - ASSERT_EQ_LOG(thread_log, new_log->get_term(), term) - auto deserialized_log = getZookeeperCreateRequest(new_log); - ASSERT_EQ_LOG(thread_log, key, deserialized_log->path) - ASSERT_EQ_LOG(thread_log, data, deserialized_log->data) - - log_index.store(log_idx, std::memory_order_release); - if (log_store->getClosedSegments().size() + 1 >= max_seg_count) - { - //remove segment - log_store->removeSegment(); - } - } - // } - // catch (std::exception & ex) - // { - // LOG_ERROR(thread_log, "thread exception : {}", ex.what()); - // } - }); - } - //LOG_INFO(log, "Max thread count {}, running {}", thread_pool.getMaxThreads(), thread_pool.active()); - thread_pool.wait(); - watch.stop(); - int mill_second = watch.elapsedMilliseconds(); - int log_size = ((key_bytes + value_bytes) + sizeof(UInt32) * 4 + sizeof(UInt32) * 6); - double total_size = 1.0 * log_size * log_count / 1000 / 1000; //M - double byte_rate = 1.0 * total_size / mill_second * 1000; - double count_rate = 1.0 * log_count / mill_second * 1000; - LOG_INFO( - log, - "Append performance : thread_count {}, size {} Byte/One Log, count {}, total_size {} M, milli second {}, byte rate {} M/S, TPS " - "{}", - thread_pool.getMaxThreads(), - log_size, - log_count, - total_size, - mill_second, - byte_rate, - count_rate); - } - cleanDirectory(log_dir); -} - - -void snapshotVolume(int last_index) -{ - Poco::Logger * log = &(Poco::Logger::get("RaftSnapshot")); - String snap_dir(SNAP_DIR + "/100"); - cleanDirectory(snap_dir); - KeeperSnapshotManager snap_mgr(snap_dir, 1000000, 20); - ptr config = cs_new(1, 0); - - RaftSettingsPtr raft_settings(RaftSettings::getDefault()); - KeeperStore storage(raft_settings->dead_session_check_period_ms); - - auto mem1 = getProcessMem(); - Stopwatch watch; - watch.start(); - UInt32 term = 1; - //UInt32 last_index = 1000000; - int value_bytes = 300; - //300 BYTE - String data; - for (int i = 0; i < value_bytes; i++) - { - data.append("v"); - } - - int thread_size = 4; - FreeThreadPool thread_pool(thread_size); - int send_count = last_index; - for (int thread_idx = 0; thread_idx < thread_size; thread_idx++) - { - thread_pool.scheduleOrThrowOnError( - [&storage, thread_idx, thread_size, send_count, &data] - { - Poco::Logger * thread_log = &(Poco::Logger::get("RaftSnapshot")); - int log_count = send_count / thread_size; - int begin = thread_idx * log_count; - int end = (thread_idx + 1) * log_count; - LOG_INFO( - thread_log, "Begin run thread {}/{}, send_count {}, range[{} - {}) ", thread_idx, thread_size, send_count, begin, end); - while (begin < end) - { - String key = std::to_string(begin + 1); - setNode(storage, key, data); - begin++; - } - }); - } - thread_pool.wait(); - /* - for (int i = 0; i < last_index; i++) - { - String key = std::to_string(i + 1); - setNode(storage, key, data); - } - */ - watch.stop(); - int mill_second = watch.elapsedMilliseconds(); - int total_size = 1.0 * (value_bytes + 100) * last_index / 1000000; //MB - double byte_rate = 1.0 * total_size / mill_second * 1000; - double count_rate = 1.0 * last_index / mill_second * 1000; - auto mem2 = getProcessMem(); - LOG_INFO( - log, - "Append log : count {}, total_size {} M, milli second {}, byte rate {} M/S, TPS {}, physicalMem {} M, virtualMem {}", - last_index, - total_size, - mill_second, - byte_rate, - count_rate, - 1.0 * (mem2.physical_mem - mem1.physical_mem) / 1000000, - 1.0 * (mem2.virtual_mem - mem1.virtual_mem) / 1000000); - - watch.start(); - snapshot meta(last_index, term, config); - size_t object_size = snap_mgr.createSnapshot(meta, storage); - (void)(object_size); - watch.stop(); - mill_second = watch.elapsedMilliseconds(); - byte_rate = 1.0 * total_size / mill_second * 1000; - count_rate = 1.0 * last_index / mill_second * 1000; - LOG_INFO( - log, - "Save snapshot : count {}, total_size {} M, milli second {}, byte rate {} M/S, TPS {}", - last_index, - total_size, - mill_second, - byte_rate, - count_rate); - - KeeperStore new_storage(raft_settings->dead_session_check_period_ms); - watch.start(); - snap_mgr.parseSnapshot(meta, new_storage); - - watch.stop(); - mill_second = watch.elapsedMilliseconds(); - byte_rate = 1.0 * total_size / mill_second * 1000; - count_rate = 1.0 * last_index / mill_second * 1000; - LOG_INFO( - log, - "Load snapshot : count {}, total_size {} M, milli second {}, byte rate {} M/S, TPS {}", - last_index, - total_size, - mill_second, - byte_rate, - count_rate); - cleanDirectory(snap_dir); -} - -int main(int argc, char ** argv) -{ - if (argc < 2) - { - std::cout << "Please run: xxx tag trace|notice|information" << std::endl; - return 0; - } - char * tag = argv[1]; - RK::TestServer app; - app.init(argc, argv); - app.run(); - if (strcmp(tag, "logSegmentThread") == 0) - { - logSegmentThread(); - } - else if (strcmp(tag, "snapshotVolume") == 0) - { - int node_size = atoi(argv[3]); - snapshotVolume(node_size); - } - return 0; -}