From 936be1cc2501c107f249aee659eed2a16ef216b4 Mon Sep 17 00:00:00 2001 From: owent Date: Wed, 11 Aug 2021 16:57:36 +0800 Subject: [PATCH 01/12] Support void for result_type --- include/design_pattern/result_type.h | 39 +++++++++-- test/case/result_type_test.cpp | 100 +++++++++++++++++++++++++++ 2 files changed, 135 insertions(+), 4 deletions(-) diff --git a/include/design_pattern/result_type.h b/include/design_pattern/result_type.h index 2a5ddc78..c65a4cea 100644 --- a/include/design_pattern/result_type.h +++ b/include/design_pattern/result_type.h @@ -47,7 +47,7 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY compact_storage_select_type; template <> struct LIBATFRAME_UTILS_API_HEAD_ONLY compact_storage_select_type { - typedef std::unique_ptr > type; + typedef std::unique_ptr> type; }; template @@ -61,18 +61,49 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY compact_storage_select_type { std::is_pod::value #endif && sizeof(T) < (sizeof(size_t) << 2), - std::unique_ptr >, typename std::unique_ptr::deleter_type>::type + std::unique_ptr>, typename std::unique_ptr::deleter_type>::type type; }; template struct LIBATFRAME_UTILS_API_HEAD_ONLY compact_storage_type; +template +struct LIBATFRAME_UTILS_API_HEAD_ONLY + compact_storage_type>> + : public std::true_type { + using value_type = void *; + using pointer = std::unique_ptr>; + using storage_type = pointer; + + static UTIL_FORCEINLINE void destroy_storage(storage_type &out) { + out.release(); + out.~storage_type(); + } + + static UTIL_FORCEINLINE void construct_storage(storage_type &out) { + // Placement new + new (reinterpret_cast(&out)) storage_type(reinterpret_cast(&out)); + } + + static UTIL_FORCEINLINE void move_storage(storage_type &out, storage_type &&in) noexcept { out = std::move(in); } + + static UTIL_FORCEINLINE void swap(storage_type &l, storage_type &r) noexcept { l.swap(r); } + + static UTIL_FORCEINLINE pointer &unref(storage_type &storage) noexcept { return storage; } + static UTIL_FORCEINLINE const pointer &unref(const storage_type &storage) noexcept { return storage; } + + static UTIL_FORCEINLINE pointer &default_instance() noexcept { + static pointer empty; + return empty; + } +}; + template struct LIBATFRAME_UTILS_API_HEAD_ONLY - compact_storage_type > > : public std::true_type { + compact_storage_type>> : public std::true_type { using value_type = T; - using pointer = std::unique_ptr >; + using pointer = std::unique_ptr>; using storage_type = std::pair; static UTIL_FORCEINLINE void destroy_storage(storage_type &out) { diff --git a/test/case/result_type_test.cpp b/test/case/result_type_test.cpp index 1d539910..44501ef0 100644 --- a/test/case/result_type_test.cpp +++ b/test/case/result_type_test.cpp @@ -458,3 +458,103 @@ CASE_TEST(result_type, non_triviall) { CASE_EXPECT_EQ(*success_obj.get_success(), "123"); CASE_EXPECT_EQ(*error_obj.get_error(), "456"); } + +CASE_TEST(result_type, has_void) { + using test_type = util::design_pattern::result_type; + auto success_obj = test_type::make_success(); + auto error_obj = test_type::make_error(456U); + + CASE_EXPECT_TRUE(success_obj.is_success()); + CASE_EXPECT_FALSE(success_obj.is_error()); + CASE_EXPECT_FALSE(success_obj.is_none()); + + CASE_EXPECT_FALSE(error_obj.is_success()); + CASE_EXPECT_TRUE(error_obj.is_error()); + CASE_EXPECT_FALSE(error_obj.is_none()); + + CASE_EXPECT_NE(success_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(success_obj.get_error().get(), nullptr); + + CASE_EXPECT_EQ(error_obj.get_success().get(), nullptr); + CASE_EXPECT_NE(error_obj.get_error().get(), nullptr); + + CASE_EXPECT_NE(success_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(*error_obj.get_error(), 456); + + { + test_type empty_obj; + CASE_EXPECT_FALSE(empty_obj.is_success()); + CASE_EXPECT_FALSE(empty_obj.is_error()); + CASE_EXPECT_TRUE(empty_obj.is_none()); + } + + // Swap success and empty + { + test_type empty_obj; + success_obj.swap(empty_obj); + + CASE_EXPECT_TRUE(empty_obj.is_success()); + CASE_EXPECT_TRUE(success_obj.is_none()); + CASE_EXPECT_NE(empty_obj.get_success().get(), nullptr); + success_obj.swap(empty_obj); + + CASE_EXPECT_TRUE(success_obj.is_success()); + CASE_EXPECT_FALSE(success_obj.is_error()); + CASE_EXPECT_FALSE(success_obj.is_none()); + } + + // Swap empty and success + { + test_type empty_obj; + empty_obj.swap(success_obj); + + CASE_EXPECT_TRUE(empty_obj.is_success()); + CASE_EXPECT_TRUE(success_obj.is_none()); + CASE_EXPECT_NE(empty_obj.get_success().get(), nullptr); + empty_obj.swap(success_obj); + + CASE_EXPECT_TRUE(success_obj.is_success()); + CASE_EXPECT_FALSE(success_obj.is_error()); + CASE_EXPECT_FALSE(success_obj.is_none()); + } + + // Swap success and error + success_obj.swap(error_obj); + + CASE_EXPECT_TRUE(error_obj.is_success()); + CASE_EXPECT_FALSE(error_obj.is_error()); + CASE_EXPECT_FALSE(error_obj.is_none()); + + CASE_EXPECT_FALSE(success_obj.is_success()); + CASE_EXPECT_TRUE(success_obj.is_error()); + CASE_EXPECT_FALSE(success_obj.is_none()); + + CASE_EXPECT_NE(error_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(error_obj.get_error().get(), nullptr); + + CASE_EXPECT_EQ(success_obj.get_success().get(), nullptr); + CASE_EXPECT_NE(success_obj.get_error().get(), nullptr); + + CASE_EXPECT_NE(error_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(*success_obj.get_error(), 456); + + // Swap error and success + success_obj.swap(error_obj); + + CASE_EXPECT_TRUE(success_obj.is_success()); + CASE_EXPECT_FALSE(success_obj.is_error()); + CASE_EXPECT_FALSE(success_obj.is_none()); + + CASE_EXPECT_FALSE(error_obj.is_success()); + CASE_EXPECT_TRUE(error_obj.is_error()); + CASE_EXPECT_FALSE(error_obj.is_none()); + + CASE_EXPECT_NE(success_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(success_obj.get_error().get(), nullptr); + + CASE_EXPECT_EQ(error_obj.get_success().get(), nullptr); + CASE_EXPECT_NE(error_obj.get_error().get(), nullptr); + + CASE_EXPECT_NE(success_obj.get_success().get(), nullptr); + CASE_EXPECT_EQ(*error_obj.get_error(), 456); +} From 7158ce55faf596da348d3607eaa05e3b117333ed Mon Sep 17 00:00:00 2001 From: owent Date: Thu, 12 Aug 2021 10:28:59 +0800 Subject: [PATCH 02/12] Add static assert for result_type --- test/case/result_type_test.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/case/result_type_test.cpp b/test/case/result_type_test.cpp index 44501ef0..4be2601a 100644 --- a/test/case/result_type_test.cpp +++ b/test/case/result_type_test.cpp @@ -14,7 +14,11 @@ CASE_TEST(result_type, all_triviall) { using test_type = util::design_pattern::result_type; auto success_obj = test_type::make_success(123); auto error_obj = test_type::make_error(456U); + static_assert( + sizeof(std::aligned_storage::type) + sizeof(int32_t) >= sizeof(test_type), + "size invalid for trivial result_type"); + CASE_MSG_INFO() << "sizeof(result_type): " << sizeof(test_type) << std::endl; CASE_EXPECT_TRUE(success_obj.is_success()); CASE_EXPECT_FALSE(success_obj.is_error()); CASE_EXPECT_FALSE(success_obj.is_none()); From c663d8116b3596054d2ea219a7f045b3310882d5 Mon Sep 17 00:00:00 2001 From: owent Date: Sun, 15 Aug 2021 22:08:20 +0800 Subject: [PATCH 03/12] wal object Signed-off-by: owent --- .../distributed_system/transaction_client.h | 24 + .../transaction_common_defs.h | 36 + .../transaction_coordinator.h | 24 + include/distributed_system/transaction_node.h | 26 + include/distributed_system/wal_client.h | 31 + include/distributed_system/wal_common_defs.h | 85 +++ include/distributed_system/wal_object.h | 637 ++++++++++++++++++ include/distributed_system/wal_publisher.h | 488 ++++++++++++++ include/distributed_system/wal_subscriber.h | 299 ++++++++ test/case/wal_object_test.cpp | 499 ++++++++++++++ test/case/wal_publisher_test.cpp | 279 ++++++++ 11 files changed, 2428 insertions(+) create mode 100644 include/distributed_system/transaction_client.h create mode 100644 include/distributed_system/transaction_common_defs.h create mode 100644 include/distributed_system/transaction_coordinator.h create mode 100644 include/distributed_system/transaction_node.h create mode 100644 include/distributed_system/wal_client.h create mode 100644 include/distributed_system/wal_common_defs.h create mode 100644 include/distributed_system/wal_object.h create mode 100644 include/distributed_system/wal_publisher.h create mode 100644 include/distributed_system/wal_subscriber.h create mode 100644 test/case/wal_object_test.cpp create mode 100644 test/case/wal_publisher_test.cpp diff --git a/include/distributed_system/transaction_client.h b/include/distributed_system/transaction_client.h new file mode 100644 index 00000000..58430c87 --- /dev/null +++ b/include/distributed_system/transaction_client.h @@ -0,0 +1,24 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Transaction client + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO(owentou): 等稳定后移入atframe_utils +namespace util { +namespace distributed_system { + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/transaction_common_defs.h b/include/distributed_system/transaction_common_defs.h new file mode 100644 index 00000000..0dd773d4 --- /dev/null +++ b/include/distributed_system/transaction_common_defs.h @@ -0,0 +1,36 @@ +// Copyright 2021 Tencent +// Created by owentou +// Common definitions for Transaction + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO(owentou): 等稳定后移入atframe_utils +namespace util { +namespace distributed_system { + +enum class transaction_status : int32_t { + kCreated = 0, // (Initial) Created and not prepare yet + kPraparing, // Running the prepare process + kApproved, // Accepted but not commited yet + kRejected, // (Final) Rejected + kCommiting, // Accepted and run commiting + kCommited, // (Final) Commited + kAborting, // Accepted but run aborting + kAborted, // (Final) Aborted +}; + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/transaction_coordinator.h b/include/distributed_system/transaction_coordinator.h new file mode 100644 index 00000000..15f42f11 --- /dev/null +++ b/include/distributed_system/transaction_coordinator.h @@ -0,0 +1,24 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Transaction coordinator + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO(owentou): 等稳定后移入atframe_utils +namespace util { +namespace distributed_system { + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/transaction_node.h b/include/distributed_system/transaction_node.h new file mode 100644 index 00000000..31a7d598 --- /dev/null +++ b/include/distributed_system/transaction_node.h @@ -0,0 +1,26 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Transaction node + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "distributed_system/transaction_common_defs.h" + +// TODO(owentou): 等稳定后移入atframe_utils +namespace util { +namespace distributed_system { + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/wal_client.h b/include/distributed_system/wal_client.h new file mode 100644 index 00000000..40ae22b2 --- /dev/null +++ b/include/distributed_system/wal_client.h @@ -0,0 +1,31 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Write Ahead Log client + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "distributed_system/wal_common_defs.h" + +namespace util { +namespace distributed_system { +/* +class LIBATFRAME_UTILS_API_HEAD_ONLY wal_client { + // TODO(owentou): send subscribe request + + private: + // TODO(owentou): load snapshot + // TODO(owentou): load logs + // TODO(owentou): dump +}; +*/ +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h new file mode 100644 index 00000000..325e917d --- /dev/null +++ b/include/distributed_system/wal_common_defs.h @@ -0,0 +1,85 @@ +// Copyright 2021 Tencent +// Created by owentou on 2021-08-12 +// Common definitions for Write Ahead Log + +#pragma once + +#include + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace util { +namespace distributed_system { + +using wal_time_point = std::chrono::system_clock::time_point; +using wal_duration = std::chrono::system_clock::duration; + +template +struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_meta_type { + wal_time_point timepoint; + LogKeyT log_key; + ActionCaseT action_case; +}; + +enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_result_code : int32_t { + kSubscriberNotFound = -201, + + kInitlization = -105, + kCallbackError = -104, + kInvalidParam = -103, + kBadLogKey = -102, + kActionNotSet = -101, + + kOk = 0, + + kIgnored = 101, + kPending = 102, + kMerge = 103, +}; + +enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_unsubscribe_reason : int32_t { + kNone = 0, + kTimeout = 1, + kClientRequest = 2, + kInvalid = 3, +}; + +template +struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_log_action_getter_trait { +#if defined(__cplusplus) && __cplusplus >= 201703L + using type = std::invoke_result_t; +#else + using type = typename std::result_of::type; +#endif +}; + +template , + class HashActionCaseT = std::hash::type>, + class EqualActionCaseT = std::equal_to::type> > +class LIBATFRAME_UTILS_API_HEAD_ONLY wal_log_operator { + public: + using log_key_type = LogKeyT; + using log_type = LogT; + using action_getter_trait = wal_log_action_getter_trait; + using action_getter_type = ActionGetter; + using action_case_type = typename action_getter_trait::type; + using log_key_compare_type = CompareLogKeyT; + using action_case_hash = HashActionCaseT; + using action_case_equal = EqualActionCaseT; + + using log_pointer = std::shared_ptr; + using log_key_result_type = util::design_pattern::result_type; +}; + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/wal_object.h b/include/distributed_system/wal_object.h new file mode 100644 index 00000000..2366ac8a --- /dev/null +++ b/include/distributed_system/wal_object.h @@ -0,0 +1,637 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Write Ahead Log + +#pragma once + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "distributed_system/wal_common_defs.h" + +#ifdef max +# undef max +#endif + +namespace util { +namespace distributed_system { + +template +class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { + public: + using storage_type = StorageT; + using private_data_type = PrivateDataT; + using log_operator_type = LogOperatorT; + + using log_type = typename log_operator_type::log_type; + using log_pointer = typename log_operator_type::log_pointer; + using log_key_type = typename log_operator_type::log_key_type; + using log_key_compare_type = typename log_operator_type::log_key_compare_type; + using action_getter_type = typename log_operator_type::action_getter_type; + using action_case_type = typename log_operator_type::action_case_type; + using log_key_result_type = typename log_operator_type::log_key_result_type; + + using log_iterator = typename std::deque::iterator; + using log_const_iterator = typename std::deque::const_iterator; + using callback_param_type = CallbackParamT; + using time_point = wal_time_point; + using duration = wal_duration; + using meta_type = wal_meta_type; + using meta_result_type = util::design_pattern::result_type; + + // Load data from storage into wal_object + using callback_load_fn_t = std::function; + + // Dump data from wal_object into storage + using callback_dump_fn_t = std::function; + + // Run log action + using callback_log_action_fn_t = std::function; + + // Log event callback + using callback_log_event_fn_t = std::function; + + // Merge action + using callback_log_merge_fn_t = + std::function; + + // Get the meta data of a log + using callback_log_get_meta_fn_t = std::function; + + // Set the meta data of a log + using callback_log_set_meta_fn_t = std::function; + + // Get the key of a log + using callback_get_log_key_fn_t = std::function; + + // Allocate a log id + using callback_alloc_log_key_fn_t = std::function; + + using action_map_type = + std::unordered_map; + + struct vtable_type { + callback_load_fn_t load; + callback_dump_fn_t dump; + callback_log_get_meta_fn_t get_meta; + callback_log_set_meta_fn_t set_meta; + callback_log_merge_fn_t merge_log; + callback_get_log_key_fn_t get_log_key; + callback_alloc_log_key_fn_t alloc_log_key; + callback_log_event_fn_t on_log_added; + callback_log_event_fn_t on_log_removed; + action_map_type delegate_action; + callback_log_action_fn_t default_action; + }; + using vtable_pointer = std::shared_ptr; + + struct congfigure_type { + duration gc_expire_duration; + size_t max_log_size; + size_t gc_log_size; + }; + using congfigure_pointer = std::shared_ptr; + + private: + UTIL_DESIGN_PATTERN_NOMOVABLE(wal_object); + UTIL_DESIGN_PATTERN_NOCOPYABLE(wal_object); + struct construct_helper { + vtable_pointer vt; + congfigure_pointer conf; + }; + + struct finally_helper { + std::function fn; + wal_object* owner; + finally_helper(wal_object& o, std::function f) : fn(f), owner(&o) {} + ~finally_helper() { + if (nullptr != owner && fn) { + fn(*owner); + } + } + }; + + public: + template + explicit wal_object(construct_helper& helper, ArgsT&&... args) + : in_log_action_callback_(false), + vtable_{helper.vt}, + configure_{helper.conf}, + private_data_(std::forward(args)...) {} + + template + static std::shared_ptr create(vtable_pointer vt, congfigure_pointer conf, ArgsT&&... args) { + if (!vt || !conf) { + return nullptr; + } + + if (!vt->get_meta || !vt->get_log_key || !vt->alloc_log_key) { + return nullptr; + } + + construct_helper helper; + helper.vt = vt; + helper.conf = conf; + return std::make_shared(helper, std::forward(args)...); + } + + static void default_configure(congfigure_type& out) { + out.gc_expire_duration = std::chrono::duration_cast(std::chrono::days{7}); + out.max_log_size = 512; + out.gc_log_size = 128; + } + + wal_result_code load(const storage_type& storage, callback_param_type param) { + if (!configure_ || !vtable_) { + return wal_result_code::kInitlization; + } + + if (!vtable_->load) { + return wal_result_code::kActionNotSet; + } + + return vtable_->load(*this, storage, param); + } + + wal_result_code dump(storage_type& storage, callback_param_type param) { + if (!configure_ || !vtable_) { + return wal_result_code::kInitlization; + } + + if (!vtable_->dump) { + return wal_result_code::kActionNotSet; + } + + return vtable_->dump(*this, storage, param); + } + + template + log_pointer allocate_log(time_point now, action_case_type action_case, callback_param_type param, ArgsT&&... args) { + if (!configure_ || !vtable_ || !vtable_->alloc_log_key || !vtable_->set_meta) { + return nullptr; + } + + log_key_result_type new_key = vtable_->alloc_log_key(*this, param); + if (!new_key.is_success()) { + return nullptr; + } + + log_pointer ret = std::make_shared(std::forward(args)...); + if (!ret) { + return ret; + } + + meta_type meta; + meta.timepoint = now; + meta.log_key = *new_key.get_success(); + meta.action_case = action_case; + vtable_->set_meta(*this, *ret, meta); + + return ret; + } + + wal_result_code emplace_back(log_pointer&& log, callback_param_type param) { + if (!log) { + return wal_result_code::kInvalidParam; + } + + if (!pending_logs_.empty() || in_log_action_callback_) { + pending_logs_.emplace_back(std::pair{std::move(log), param}); + return wal_result_code::kPending; + } + + in_log_action_callback_ = true; + finally_helper guard(*this, [](wal_object& self) { self.in_log_action_callback_ = false; }); + + wal_result_code ret; + if (!vtable_ || !vtable_->get_log_key) { + ret = pusk_back_inner(std::move(log), param); + } else { + log_key_type this_key = vtable_->get_log_key(*this, *log); + if (global_ingore_ && log_key_compare_(this_key, *global_ingore_)) { + ret = wal_result_code::kIgnored; + } else { + ret = pusk_back_inner(std::move(log), param); + } + } + + while (!pending_logs_.empty()) { + auto pending_log = pending_logs_.front(); + pending_logs_.pop_front(); + + if (!pending_log.first) { + continue; + } + + if (vtable_ && vtable_->get_log_key) { + log_key_type this_key = vtable_->get_log_key(*this, *pending_log.first); + if (global_ingore_ && log_key_compare_(this_key, *global_ingore_)) { + continue; + } + } + + pusk_back_inner(std::move(pending_log.first), pending_log.second); + } + + return ret; + } + + wal_result_code push_back(log_pointer log, callback_param_type param) { return emplace_back(std::move(log), param); } + + wal_result_code remove_before(time_point now, size_t max_count = std::numeric_limits::max()) { + if (!vtable_ || !vtable_->get_meta) { + return wal_result_code::kInitlization; + } + + for (size_t count = 0; count < max_count; ++count) { + if (logs_.empty()) { + break; + } + + meta_result_type meta = vtable_->get_meta(*this, *logs_.back()); + if (meta.is_success() && meta.get_success()->timepoint < now) { + pop_front_inner(); + } else if (meta.is_error()) { + return *meta.get_error(); + } else if (meta.is_none()) { + return wal_result_code::kCallbackError; + } else { + break; + } + } + + return wal_result_code::kOk; + } + + /** + * @brief Clean and remove expired logs + * + * @param now current time point + * @param hold Do not remove logs with key greater than `hold` + * @param max_count Max log count to remove + * @return size_t Log count removed + */ + size_t gc(time_point now, const log_key_type* hold = nullptr, size_t max_count = std::numeric_limits::max()) { + duration gc_expire_duration = std::chrono::duration_cast(std::chrono::days{7}); + size_t max_log_size = 512; + size_t gc_log_size = 128; + if (configure_) { + if (configure_->gc_log_size > 0) { + gc_log_size = configure_->gc_log_size; + } + if (configure_->max_log_size > 0) { + max_log_size = configure_->max_log_size; + } + if (configure_->gc_expire_duration > duration::zero()) { + gc_expire_duration = configure_->gc_expire_duration; + } + } + size_t ret = 0; + for (; ret < max_count; ++ret) { + if (logs_.empty()) { + break; + } + + if (logs_.size() <= gc_log_size) { + break; + } + + if (logs_.size() > max_log_size) { + pop_front_inner(); + continue; + } + + if (!vtable_ || !vtable_->get_meta) { + pop_front_inner(); + continue; + } + + meta_result_type meta = vtable_->get_meta(*this, **logs_.begin()); + if (meta.is_error() || meta.is_none()) { + pop_front_inner(); + } else if (meta.is_success() && meta.get_success()->timepoint + gc_expire_duration <= now) { + if (nullptr != hold && log_key_compare_(*hold, meta.get_success()->log_key)) { + break; + } + pop_front_inner(); + } else { + break; + } + } + + return ret; + } + + const std::unique_ptr& get_global_ingore_key() const noexcept { return global_ingore_; } + + template + void set_global_ingore_key(ToKey&& key) { + if (global_ingore_) { + *global_ingore_ = std::forward(key); + } else { + global_ingore_.reset(new log_key_type{std::forward(key)}); + } + } + + const std::unique_ptr& get_last_removed_key() const noexcept { return global_last_removed_; } + + template + void set_last_removed_key(ToKey&& key) { + if (global_last_removed_) { + *global_last_removed_ = std::forward(key); + } else { + global_last_removed_.reset(new log_key_type{std::forward(key)}); + } + } + + inline const std::deque& get_all_logs() const noexcept { return logs_; } + + inline const private_data_type& get_private_data() const noexcept { return private_data_; } + inline private_data_type& get_private_data() noexcept { return private_data_; } + + inline const log_key_compare_type& get_log_key_compare() const noexcept { return log_key_compare_; } + inline log_key_compare_type& get_log_key_compare() noexcept { return log_key_compare_; } + + inline const congfigure_type& get_configure() const noexcept { + // We can not create wal_object without congfigure, so it's safe here + return *configure_; + } + + inline congfigure_type& get_configure() noexcept { + // We can not create wal_object without congfigure, so it's safe here + return *configure_; + } + + log_pointer find_log(const log_key_type& key) noexcept { + log_iterator iter = log_lower_bound(key); + if (iter == logs_.end()) { + return nullptr; + } + + log_key_type found_key = vtable_->get_log_key(*this, **iter); + if (!log_key_compare_(key, found_key) && !log_key_compare_(found_key, key)) { + return *iter; + } + + return nullptr; + } + + log_const_iterator find_log(const log_key_type& key) const noexcept { + log_const_iterator iter = log_lower_bound(key); + if (iter == logs_.end()) { + return nullptr; + } + + log_key_type found_key = vtable_->get_log_key(*this, **iter); + if (!log_key_compare_(key, found_key) && !log_key_compare_(found_key, key)) { + return *iter; + } + + return nullptr; + } + + inline log_iterator log_begin() noexcept { return logs_.begin(); } + inline log_iterator log_end() noexcept { return logs_.end(); } + inline log_const_iterator log_cbegin() const noexcept { return logs_.begin(); } + inline log_const_iterator log_cend() const noexcept { return logs_.end(); } + + log_iterator log_lower_bound(const log_key_type& key) noexcept { + if (!vtable_ || !vtable_->get_log_key) { + return logs_.end(); + } + + if (logs_.empty()) { + return logs_.end(); + } else { + // Optimization for nothing + // The most frequently usage of this function is used to renew subscriber, which already has the latest log + log_key_type last_key = vtable_->get_log_key(*this, **logs_.rbegin()); + if (log_key_compare_(last_key, key)) { + return logs_.end(); + } + } + + return std::lower_bound(logs_.begin(), logs_.end(), key, [this](const log_pointer& l, const log_key_type& r) { + log_key_type log_key = this->vtable_->get_log_key(*this, *l); + return this->log_key_compare_(log_key, r); + }); + } + + log_const_iterator log_lower_bound(const log_key_type& key) const noexcept { + if (!vtable_ || !vtable_->get_log_key) { + return logs_.end(); + } + + if (logs_.empty()) { + return logs_.end(); + } else { + // Optimization for nothing + // The most frequently usage of this function is used to renew subscriber, which already has the latest log + log_key_type last_key = vtable_->get_log_key(*this, **logs_.rbegin()); + if (log_key_compare_(last_key, key)) { + return logs_.end(); + } + } + + return std::lower_bound(logs_.begin(), logs_.end(), key, [this](const log_pointer& l, const log_key_type& r) { + log_key_type log_key = this->vtable_->get_log_key(*this, *l); + return this->log_key_compare_(log_key, r); + }); + } + + log_iterator log_upper_bound(const log_key_type& key) noexcept { + if (!vtable_ || !vtable_->get_log_key) { + return logs_.end(); + } + + if (logs_.empty()) { + return logs_.end(); + } else { + // Optimization for nothing + // The most frequently usage of this function is used to renew subscriber, which already has the latest log + log_key_type last_key = vtable_->get_log_key(*this, **logs_.rbegin()); + if (!log_key_compare_(key, last_key)) { + return logs_.end(); + } + } + + return std::upper_bound(logs_.begin(), logs_.end(), key, [this](const log_key_type& l, const log_pointer& r) { + log_key_type log_key = this->vtable_->get_log_key(*this, *r); + return this->log_key_compare_(l, log_key); + }); + } + + log_const_iterator log_upper_bound(const log_key_type& key) const noexcept { + if (!vtable_ || !vtable_->get_log_key) { + return logs_.end(); + } + + if (logs_.empty()) { + return logs_.end(); + } else { + // Optimization for nothing + // The most frequently usage of this function is used to renew subscriber, which already has the latest log + log_key_type last_key = vtable_->get_log_key(*this, **logs_.rbegin()); + if (!log_key_compare_(key, last_key)) { + return logs_.end(); + } + } + + return std::upper_bound(logs_.begin(), logs_.end(), key, [this](const log_key_type& l, const log_pointer& r) { + log_key_type log_key = this->vtable_->get_log_key(*this, *r); + return this->log_key_compare_(l, log_key); + }); + } + + std::pair log_all_range() noexcept { + return std::pair(logs_.begin(), logs_.end()); + } + + std::pair log_all_range() const noexcept { + return std::pair(logs_.begin(), logs_.end()); + } + + private: + wal_result_code redo_log(const log_pointer& log, callback_param_type param) { + if (!log) { + return wal_result_code::kInvalidParam; + } + + if (!vtable_ || !vtable_->get_meta) { + return wal_result_code::kInitlization; + } + + meta_result_type meta = vtable_->get_meta(*this, *log); + if (meta.is_error()) { + return *meta.get_error(); + } else if (!meta.is_success()) { + return wal_result_code::kCallbackError; + } + + auto iter = vtable_->delegate_action.find(meta.get_success()->action_case); + if (iter != vtable_->delegate_action.end()) { + return iter->second(*this, *log, param); + } else if (vtable_->default_action) { + return vtable_->default_action(*this, *log, param); + } else { + return wal_result_code::kActionNotSet; + } + } + + wal_result_code pusk_back_inner_uncheck(log_pointer&& log, callback_param_type param) { + wal_result_code ret = redo_log(log, param); + if (wal_result_code::kOk != ret) { + return ret; + } + logs_.push_back(log); + if (vtable_ && vtable_->on_log_added) { + vtable_->on_log_added(*this, log); + } + + return ret; + } + + wal_result_code pusk_back_inner(log_pointer&& log, callback_param_type param) { + if (!log) { + return wal_result_code::kInvalidParam; + } + + if (!vtable_) { + return wal_result_code::kInvalidParam; + } + + // Empty + if (logs_.empty()) { + return pusk_back_inner_uncheck(std::move(log), param); + } + + // Can not get log key + if (!vtable_ || !vtable_->get_log_key) { + return pusk_back_inner_uncheck(std::move(log), param); + } + + log_key_type last_key = vtable_->get_log_key(*this, *logs_.back()); + log_key_type this_key = vtable_->get_log_key(*this, *log); + // Has log key + // -- push_back + if (log_key_compare_(last_key, this_key)) { + return pusk_back_inner_uncheck(std::move(log), param); + } + // -- insert + auto iter = + std::lower_bound(logs_.begin(), logs_.end(), this_key, [this](const log_pointer& l, const log_key_type& r) { + log_key_type log_key = this->vtable_->get_log_key(*this, *l); + return this->log_key_compare_(log_key, r); + }); + + if (iter != logs_.end()) { + last_key = vtable_->get_log_key(*this, *logs_.back()); + if (!log_key_compare_(last_key, this_key) && !log_key_compare_(this_key, last_key)) { + if (vtable_->merge_log) { + vtable_->merge_log(*this, param, **iter, *log); + } + return wal_result_code::kMerge; + } + } + wal_result_code ret = redo_log(log, param); + if (wal_result_code::kOk != ret) { + return ret; + } + logs_.insert(iter, log); + if (vtable_ && vtable_->on_log_added) { + vtable_->on_log_added(*this, log); + } + return ret; + } + + void pop_front_inner() { + if (logs_.empty()) { + return; + } + + log_pointer log = logs_.front(); + logs_.pop_front(); + + if (vtable_ && vtable_->get_log_key) { + log_key_type key = vtable_->get_log_key(*this, *log); + if (!(global_last_removed_ && log_key_compare_(key, *global_last_removed_))) { + set_last_removed_key(std::move(key)); + } + } + + if (vtable_ && vtable_->on_log_removed) { + vtable_->on_log_removed(*this, log); + } + } + + private: + bool in_log_action_callback_; + vtable_pointer vtable_; + congfigure_pointer configure_; + private_data_type private_data_; + log_key_compare_type log_key_compare_; + + // global + std::unique_ptr global_last_removed_; // ignore all log lower than this key + std::unique_ptr global_ingore_; // ignore all log lower than this key + + // logs(libstdc++ is 512Byte for each block and maintain block index just like std::vector) + std::deque logs_; + std::list > pending_logs_; +}; + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/wal_publisher.h b/include/distributed_system/wal_publisher.h new file mode 100644 index 00000000..8c5c0a3d --- /dev/null +++ b/include/distributed_system/wal_publisher.h @@ -0,0 +1,488 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Write Ahead Log publisher + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "distributed_system/wal_common_defs.h" +#include "distributed_system/wal_object.h" +#include "distributed_system/wal_subscriber.h" + +namespace util { +namespace distributed_system { + +template +class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { + public: + using log_operator_type = LogOperatorT; + using object_type = wal_object; + using subscriber_type = WalSubscriber; + + using storage_type = typename object_type::storage_type; + using log_type = typename log_operator_type::log_type; + using log_pointer = typename log_operator_type::log_pointer; + using log_key_type = typename log_operator_type::log_key_type; + using log_key_compare_type = typename log_operator_type::log_key_compare_type; + using action_getter_type = typename log_operator_type::action_getter_type; + using action_case_type = typename log_operator_type::action_case_type; + using log_key_result_type = typename log_operator_type::log_key_result_type; + + using log_iterator = typename object_type::log_iterator; + using log_const_iterator = typename object_type::log_const_iterator; + using callback_param_type = typename object_type::callback_param_type; + using time_point = typename object_type::time_point; + using duration = typename object_type::duration; + using meta_type = typename object_type::meta_type; + using meta_result_type = typename object_type::meta_result_type; + + using private_data_type = typename object_type::private_data_type; + + using subscriber_key_type = typename subscriber_type::key_type; + using subscriber_iterator = typename subscriber_type::subscriber_iterator; + using subscriber_const_iterator = typename subscriber_type::subscriber_const_iterator; + using subscriber_pointer = typename subscriber_type::pointer; + using subscriber_private_data_type = typename subscriber_type::private_data_type; + using subscriber_manager_type = typename subscriber_type::manager; + + using callback_load_fn_t = typename object_type::callback_load_fn_t; + using callback_dump_fn_t = typename object_type::callback_dump_fn_t; + using callback_log_action_fn_t = typename object_type::callback_log_action_fn_t; + using callback_log_event_fn_t = typename object_type::callback_log_event_fn_t; + using callback_log_merge_fn_t = typename object_type::callback_log_merge_fn_t; + using callback_log_get_meta_fn_t = typename object_type::callback_log_get_meta_fn_t; + using callback_log_set_meta_fn_t = typename object_type::callback_log_set_meta_fn_t; + using callback_get_log_key_fn_t = typename object_type::callback_get_log_key_fn_t; + using callback_alloc_log_key_fn_t = typename object_type::callback_alloc_log_key_fn_t; + + // Send snapshot to a subscriber clients + using callback_send_snapshot_fn_t = + std::function; + + // Send logs to a subscriber clients + using callback_send_logs_fn_t = + std::function; + + // Send subscribe response + using callback_send_subscribe_response_fn_t = + std::function; + + // Send subscribe response + using callback_check_subscriber_fn_t = + std::function; + + // On subscriber added + using callback_on_subscriber_added_fn_t = + std::function; + + // On subscriber removed + using callback_on_subscriber_removed_fn_t = + std::function; + + struct vtable_type : public object_type::vtable_type { + callback_send_snapshot_fn_t send_snapshot; + callback_send_logs_fn_t send_logs; + callback_send_subscribe_response_fn_t subscribe_response; + + callback_check_subscriber_fn_t check_subscriber; + callback_on_subscriber_added_fn_t on_subscriber_added; + callback_on_subscriber_removed_fn_t on_subscriber_removed; + }; + using vtable_pointer = std::shared_ptr; + + struct congfigure_type : public object_type::congfigure_type { + duration subscriber_timeout; + }; + using congfigure_pointer = std::shared_ptr; + + private: + UTIL_DESIGN_PATTERN_NOMOVABLE(wal_publisher); + UTIL_DESIGN_PATTERN_NOCOPYABLE(wal_publisher); + struct construct_helper { + vtable_pointer vt; + congfigure_pointer conf; + std::shared_ptr wal_object; + std::shared_ptr subscriber_manager; + }; + + public: + template + explicit wal_publisher(construct_helper& helper, ArgsT&&... args) + : vtable_(helper.vt), + configure_(helper.conf), + wal_object_(helper.wal_object), + subscriber_manager_(helper.subscriber_manager) {} + + template + static std::shared_ptr create(vtable_pointer vt, congfigure_pointer conf, ArgsT&&... args) { + if (!vt || !conf) { + return nullptr; + } + + if (!vt->get_meta || !vt->get_log_key || !vt->alloc_log_key) { + return nullptr; + } + + if (!vt->send_snapshot || !vt->send_logs) { + return nullptr; + } + + construct_helper helper; + helper.vt = vt; + helper.conf = conf; + helper.wal_object = object_type::create( + std::static_pointer_cast(helper.vt), + std::static_pointer_cast(helper.conf), std::forward(args)...); + helper.subscriber_manager = std::make_shared(); + if (!helper.wal_object || !helper.subscriber_manager) { + return nullptr; + } + + return std::make_shared(helper, std::forward(args)...); + } + + static congfigure_pointer make_configure() { + congfigure_pointer ret = std::make_shared(); + if (!ret) { + return ret; + } + + object_type::default_configure(*ret); + ret->subscriber_timeout = std::chrono::duration_cast(std::chrono::minutes{10}); + return ret; + } + + wal_result_code load(const storage_type& storage, callback_param_type param) { + if (!wal_object_) { + return wal_result_code::kInitlization; + } + + return wal_object_->load(storage, param); + } + + wal_result_code dump(storage_type& storage, callback_param_type param) { + if (!wal_object_) { + return wal_result_code::kInitlization; + } + + return wal_object_->dump(storage, param); + } + + template + log_pointer allocate_log(time_point now, action_case_type action_case, callback_param_type param, ArgsT&&... args) { + if (!wal_object_) { + return nullptr; + } + + return wal_object_->allocate_log(now, action_case, param, std::forward(args)); + } + + wal_result_code emplace_back_log(log_pointer&& log, callback_param_type param) { + if (!wal_object_) { + return wal_result_code::kInitlization; + } + + return wal_object_->emplace_back(std::move(log), param); + } + + wal_result_code push_back_log(log_pointer log, callback_param_type param) { + if (!wal_object_) { + return wal_result_code::kInitlization; + } + + return wal_object_->emplace_back(std::move(log), param); + } + + const std::unique_ptr& get_global_log_ingore_key() const noexcept { + if (!wal_object_) { + return nullptr; + } + return wal_object_->get_global_ingore_key(); + } + + template + void set_global_log_ingore_key(ToKey&& key) { + if (!wal_object_) { + return; + } + + wal_object_->set_global_ingore_key(std::forward(key)); + } + + log_pointer find_log(const log_key_type& key) noexcept { + if (!wal_object_) { + return nullptr; + } + + return wal_object_->find_log(key); + } + + log_const_iterator find_log(const log_key_type& key) const noexcept { + if (!wal_object_) { + return nullptr; + } + + return wal_object_->find_log(key); + } + + inline const private_data_type& get_private_data() const noexcept { return wal_object_->get_private_data(); } + inline private_data_type& get_private_data() noexcept { return wal_object_->get_private_data(); } + + inline const log_key_compare_type& get_log_key_compare() const noexcept { return wal_object_->get_log_key_compare(); } + inline log_key_compare_type& get_log_key_compare() noexcept { return wal_object_->get_log_key_compare(); } + + inline const congfigure_type& get_configure() const noexcept { + // We can not create wal_object without congfigure, so it's safe here + return *configure_; + } + + inline congfigure_type& get_configure() noexcept { + // We can not create wal_object without congfigure, so it's safe here + return *configure_; + } + + const object_type& get_log_manager() const noexcept { return *wal_object_; } + object_type& get_log_manager() noexcept { return *wal_object_; } + + const subscriber_manager_type& get_subscribe_manager() const noexcept { return *subscriber_manager_; } + subscriber_manager_type& get_subscribe_manager() noexcept { return *subscriber_manager_; } + + subscriber_pointer find_subscriber(const subscriber_key_type& key, callback_param_type param) { + subscriber_pointer ret = subscriber_manager_->find(key); + + if (ret && !check_subscriber(ret, param)) { + ret.reset(); + } + + return ret; + } + + bool check_subscriber(const subscriber_pointer& subscriber, callback_param_type param) { + if (!subscriber) { + return false; + } + + if (vtable_ && vtable_->check_subscriber) { + if (!vtable_->check_subscriber(*this, subscriber, param)) { + remove_subscriber(subscriber, wal_unsubscribe_reason::kInvalid, std::move(param)); + return false; + } + } + + return true; + } + + template + subscriber_pointer create_subscriber(const subscriber_key_type& key, const time_point& now, callback_param_type param, + ArgsT&&... args) { + subscriber_pointer subscriber = find_subscriber(key); + if (subscriber) { + subscriber->set_heartbeat_timeout(configure_->subscriber_timeout); + subscriber_manager_->subscribe(*subscriber, now); + return subscriber; + } + + subscriber = subscriber_manager_->create(key, now, configure_->subscriber_timeout, std::forward(args)...); + if (subscriber && vtable_) { + if (vtable_->on_subscriber_added) { + vtable_->on_subscriber_added(*this, subscriber, param); + } + + auto range = subscriber_manager_->find_iterator(key); + if (range.first != range.second) { + send_snapshot(range.first, range.second, std::move(param)); + } + } + } + + void remove_subscriber(const subscriber_key_type& key, wal_unsubscribe_reason reason, callback_param_type param) { + subscriber_pointer subscriber = subscriber_manager_->unsubscribe(key, reason); + if (subscriber && vtable_ && vtable_->on_subscriber_removed) { + vtable_->on_subscriber_removed(this, subscriber, reason, param); + } + } + + void remove_subscriber(const subscriber_pointer& checked, wal_unsubscribe_reason reason, callback_param_type param) { + subscriber_pointer subscriber = subscriber_manager_->unsubscribe(checked, reason); + if (subscriber && vtable_ && vtable_->on_subscriber_removed) { + vtable_->on_subscriber_removed(this, subscriber, reason, param); + } + } + + size_t tick(const time_point& now, callback_param_type param, size_t max_event = std::numeric_limits::max()) { + size_t ret = 0; + if (0 == max_event) { + max_event = std::numeric_limits::max(); + } + + bool has_event = true; + while (ret < max_event && has_event) { + size_t round = max_event; + has_event = false; + if (max_event > 16) { + round = max_event / 16; + } + + // GC logs + size_t res = wal_object_->gc(now, &broadcast_key_bound_, round); + if (res > 0) { + has_event = true; + ret += res; + } + + // Subscriber expires + for (size_t j = 0; j < round; ++j) { + subscriber_pointer subscriber = subscriber_manager_->get_first_expired(now); + if (subscriber) { + remove_subscriber(subscriber, wal_unsubscribe_reason::kTimeout, param); + ++ret; + has_event = true; + } else { + break; + } + } + + // broadcast logs + res = broadcast(param); + if (res > 0) { + has_event = true; + ret += res; + } + } + + return ret; + } + + wal_result_code receive_subscribe(const subscriber_key_type& key, log_key_type last_checkpoint, const time_point& now, + callback_param_type param) { + subscriber_pointer subscriber = find_subscriber(key, param); + if (!subscriber) { + return wal_result_code::kSubscriberNotFound; + } + + // Reset timer + subscriber_manager_->reset_timer(subscriber, now); + + if (wal_object_->get_last_removed_key()) { + // Some log can not be resend, send snapshot + if (wal_object_->get_log_key_compare()(last_checkpoint, *wal_object_->get_last_removed_key())) { + auto iters = subscriber_manager_->find_iterator(key); + auto notify_result = send_snapshot(iters.first, iters.second, std::move(param)); + return send_subscribe_response(subscriber, notify_result, param); + } + } + + log_const_iterator log_iter = wal_object_->log_upper_bound(key); + if (log_iter != wal_object_->log_cend()) { + auto iters = subscriber_manager_->find_iterator(key); + auto notify_result = send_logs(log_iter, wal_object_->log_cend(), iters.first, iters.second, std::move(param)); + return send_subscribe_response(subscriber, notify_result, param); + } + + return send_subscribe_response(subscriber, wal_result_code::kOk, param); + } + + template + size_t broadcast(ParamT&& param) { + if (!vtable_ || !vtable_->get_log_key) { + return 0; + } + + // No more to broadcast + std::pair logs; + if (broadcast_key_bound_) { + logs = std::pair(wal_object_->log_upper_bound(*broadcast_key_bound_), + wal_object_->log_cend()); + } else { + logs = wal_object_->log_all_range(); + } + if (logs.first == logs.second) { + return 0; + } + + std::pair subscribers = subscriber_manager_->all_range(); + if (subscribers.first != subscribers.second) { + send_logs(logs.first, logs.second, subscribers.first, subscribers.second, std::forward(param)); + } + + int ret = 0; + log_const_iterator last; + while (logs.first != logs.second) { + last = logs.first; + ++logs.first; + ++ret; + } + + if (broadcast_key_bound_) { + *broadcast_key_bound_ = vtable_->get_log_key(*wal_object_, **last); + } else { + broadcast_key_bound_.reset(new log_key_type{vtable_->get_log_key(*wal_object_, **last)}); + } + + return ret; + } + + template + wal_result_code send_snapshot(subscriber_iterator begin, subscriber_iterator end, ParamT&& param) { + if (!vtable_ || !vtable_->send_snapshot) { + return wal_result_code::kActionNotSet; + } + + if (begin == end) { + return wal_result_code::kOk; + } + + return vtable_->send_snapshot(*this, begin, end, std::forward(param)); + } + + template + wal_result_code send_logs(log_const_iterator log_begin, log_const_iterator log_end, subscriber_iterator sub_begin, + subscriber_iterator sub_end, ParamT&& param) { + if (!vtable_ || !vtable_->send_logs) { + return wal_result_code::kActionNotSet; + } + + if (log_begin == log_end || sub_end == sub_begin) { + return wal_result_code::kOk; + } + + return vtable_->send_logs(*this, log_begin, log_end, sub_begin, sub_end, std::forward(param)); + } + + template + wal_result_code send_subscribe_response(const subscriber_pointer& subscriber, wal_result_code code, ParamT&& param) { + if (!subscriber) { + return wal_result_code::kSubscriberNotFound; + } + + if (!vtable_ || !vtable_->subscribe_response) { + return code; + } + + return vtable_->subscribe_response(*this, subscriber, code, std::forward(param)); + } + + private: + vtable_pointer vtable_; + congfigure_pointer configure_; + + // logs + std::shared_ptr wal_object_; + std::shared_ptr subscriber_manager_; + + // publish-subscribe + std::unique_ptr broadcast_key_bound_; +}; + +} // namespace distributed_system +} // namespace util diff --git a/include/distributed_system/wal_subscriber.h b/include/distributed_system/wal_subscriber.h new file mode 100644 index 00000000..7cf5f51d --- /dev/null +++ b/include/distributed_system/wal_subscriber.h @@ -0,0 +1,299 @@ +// Copyright 2021 Tencent +// Created by owentou +// Stanards operations for Write Ahead Log subscriber + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "distributed_system/wal_common_defs.h" + +namespace util { +namespace distributed_system { + +template , + class EqualActionCaseT = std::equal_to > +class LIBATFRAME_UTILS_API_HEAD_ONLY wal_subscriber { + public: + using pointer = std::shared_ptr; + + using private_data_type = PrivateDataT; + + using key_type = KeyT; + using key_hash = HashActionCaseT; + using key_equal = EqualActionCaseT; + using time_point = wal_time_point; + using duration = wal_duration; + + using subscriber_collector_type = std::unordered_map; + using subscriber_iterator = typename subscriber_collector_type::iterator; + using subscriber_const_iterator = typename subscriber_collector_type::const_iterator; + + struct timer_type { + time_point timeout; + std::weak_ptr subscriber; + }; + + private: + UTIL_DESIGN_PATTERN_NOMOVABLE(wal_subscriber); + UTIL_DESIGN_PATTERN_NOCOPYABLE(wal_subscriber); + struct construct_helper {}; + + friend class manager; + + public: + class manager { + private: + UTIL_DESIGN_PATTERN_NOMOVABLE(manager); + UTIL_DESIGN_PATTERN_NOCOPYABLE(manager); + + void remove_subscriber_timer(std::list::iterator& out) { + if (out == subscribers_timer_.end()) { + return; + } + + subscribers_timer_.erase(out); + out = subscribers_timer_.end(); + } + + void insert_subscriber_timer(const time_point& now, const pointer& subscriber) { + if (!subscriber) { + return; + } + + if (this != subscriber->owner_) { + return; + } + + if (subscriber->timer_iter_ != subscribers_timer_.end()) { + remove_subscriber_timer(subscriber->timer_iter_); + } + + timer_type timer; + timer.timeout = now + subscriber->get_heartbeat_timeout(); + timer.subscriber = subscriber; + + subscriber->timer_iter_ = subscribers_timer_.insert(subscribers_timer_.end(), timer); + } + + void unbind_owner(wal_subscriber& subscriber) { + if (nullptr != subscriber.owner_) { + subscriber.owner_->remove_subscriber_timer(subscriber.timer_iter_); + subscriber.owner_ = nullptr; + } + } + + public: + manager() {} + + void reset_timer(const pointer& subscriber, const time_point& now) { + if (!subscriber) { + return; + } + + if (subscriber->owner_ != this) { + return; + } + + remove_subscriber_timer(subscriber->timer_iter_); + insert_subscriber_timer(now, subscriber); + } + + void subscribe(const pointer& subscriber, const time_point& now) { + if (!subscriber) { + return; + } + + if (subscriber->owner_ != this) { + return; + } + + subscriber->update_heartbeat_time_point(now); + reset_timer(subscriber, now); + } + + pointer unsubscribe(const key_type& key, wal_unsubscribe_reason) { + auto iter = subscribers_.find(key); + if (iter == subscribers_.end()) { + return nullptr; + } + + pointer subscriber = iter->second; + // Remove from collector + subscribers_.erase(iter); + + // Unbind + if (subscriber) { + unbind_owner(*subscriber); + } + + return subscriber; + } + + pointer unsubscribe(const pointer& subscriber, wal_unsubscribe_reason) { + auto iter = subscribers_.find(subscriber->get_key()); + if (iter == subscribers_.end()) { + return nullptr; + } + + if (iter->second != subscriber) { + return nullptr; + } + + // Remove from collector + subscribers_.erase(iter); + + // Unbind + if (subscriber) { + unbind_owner(*subscriber); + } + + return subscriber; + } + + template + pointer create(const key_type& key, const time_point& now, const duration& timeout, ArgsT&&... args) { + auto old = subscribers_.find(key); + if (old != subscribers_.end()) { + if (!old->second) { + subscribers_.erase(old); + } else { + old->second->update_heartbeat_time_point(now); + old->second->set_heartbeat_timeout(timeout); + return old->second; + } + } + + construct_helper guard; + auto ret = std::make_shared(guard, *this, key, now, timeout, subscribers_timer_.end(), + std::forward(args)...); + if (!ret) { + return ret; + } + + subscribers_[key] = ret; + insert_subscriber_timer(now, ret); + + return ret; + } + + pointer find(const key_type& key) noexcept { + auto iter = subscribers_.find(key); + if (iter == subscribers_.end()) { + return nullptr; + } + + return iter->second; + } + + std::pair find_iterator(const key_type& key) noexcept { + auto iter = subscribers_.find(key); + if (iter == subscribers_.end()) { + return std::pair(subscribers_.end(), subscribers_.end()); + } + + subscriber_iterator end = iter; + ++end; + return std::pair(iter, end); + } + + std::pair find_iterator(const key_type& key) const noexcept { + auto iter = subscribers_.find(key); + if (iter == subscribers_.end()) { + return std::pair(subscribers_.end(), subscribers_.end()); + } + + subscriber_const_iterator end = iter; + ++end; + return std::pair(iter, end); + } + + std::pair all_range() noexcept { + return std::pair(subscribers_.begin(), subscribers_.end()); + } + + std::pair all_range() const noexcept { + return std::pair(subscribers_.begin(), subscribers_.end()); + } + + pointer get_first_expired(const time_point& now) { + while (!subscribers_timer_.empty()) { + auto begin = subscribers_timer_.begin(); + if ((*begin).timeout >= now) { + break; + } + + if ((*begin).subscriber.expired()) { + subscribers_timer_.erase(begin); + continue; + } + + pointer subscriber = (*begin).subscriber.lock(); + if (!subscriber) { + subscribers_timer_.erase(begin); + continue; + } + + // duplicated iterator + if (subscriber->timer_iter_ != begin) { + subscribers_timer_.erase(begin); + continue; + } + + return subscriber; + } + + return nullptr; + } + + private: + std::list subscribers_timer_; + subscriber_collector_type subscribers_; + }; + + public: + template + wal_subscriber(construct_helper&, manager& owner, const key_type& key, const time_point& now, const duration& timeout, + std::list::iterator timer_iter, ArgsT&&... args) + : owner_(&owner), + key_(key), + last_heartbeat_timepoint_(now), + heartbeat_timeout_(timeout), + private_data_{std::forward(args)...}, + timer_iter_(timer_iter) {} + + inline const key_type& get_key() const noexcept { return key_; } + + inline const private_data_type& get_private_data() const noexcept { return private_data_; } + inline private_data_type& get_private_data() noexcept { return private_data_; } + + inline duration get_heartbeat_timeout() const noexcept { return duration; } + inline void set_heartbeat_timeout(duration timeout) noexcept { heartbeat_timeout_ = timeout; } + + inline time_point get_last_heartbeat_time_point() const noexcept { return last_heartbeat_timepoint_; } + inline void update_heartbeat_time_point(time_point tp) noexcept { last_heartbeat_timepoint_ = tp; } + + inline bool is_offline(const time_point& now) const noexcept { + return last_heartbeat_timepoint_ + heartbeat_timeout_ >= now; + } + + private: + manager* owner_; + key_type key_; + time_point last_heartbeat_timepoint_; + duration heartbeat_timeout_; + private_data_type private_data_; + + std::list::iterator timer_iter_; +}; + +} // namespace distributed_system +} // namespace util diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp new file mode 100644 index 00000000..12096a8e --- /dev/null +++ b/test/case/wal_object_test.cpp @@ -0,0 +1,499 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "frame/test_macros.h" + +enum class test_wal_object_log_action { + kDoNothing = 0, + kRecursivePushBack, + kFallbackDefault, +}; + +struct test_wal_object_log_type { + util::distributed_system::wal_time_point timepoint; + int64_t log_key; + test_wal_object_log_action action; + int64_t data; +}; + +struct test_wal_object_log_storage_type { + std::vector logs; + int64_t global_ignore; +}; + +struct test_wal_object_log_action_getter { + test_wal_object_log_action operator()(const test_wal_object_log_type& log) { return log.action; } +}; + +struct test_wal_object_context {}; + +struct test_wal_object_private_type { + test_wal_object_log_storage_type* storage; +}; + +using test_wal_object_log_operator = + util::distributed_system::wal_log_operator; +using test_wal_object_type = + util::distributed_system::wal_object; + +struct test_wal_object_stats { + int64_t key_alloc; + size_t merge_count; + size_t delegate_action_count; + size_t default_action_count; + size_t event_on_log_added; + size_t event_on_log_removed; + + test_wal_object_type::log_type last_log; +}; + +namespace details { +test_wal_object_stats g_test_wal_object_stats{1, 0, 0, 0, 0, 0}; +} + +static test_wal_object_type::vtable_pointer create_vtable() { + using wal_object_type = test_wal_object_type; + using wal_result_code = util::distributed_system::wal_result_code; + + wal_object_type::vtable_pointer ret = std::make_shared(); + + ret->load = [](wal_object_type& wal, const wal_object_type::storage_type& from, + wal_object_type::callback_param_type) -> wal_result_code { + *wal.get_private_data().storage = from; + wal.set_global_ingore_key(from.global_ignore); + return wal_result_code::kOk; + }; + + ret->dump = [](const wal_object_type& wal, wal_object_type::storage_type& to, + wal_object_type::callback_param_type) -> wal_result_code { + to = *wal.get_private_data().storage; + if (wal.get_global_ingore_key()) { + to.global_ignore = *wal.get_global_ingore_key(); + } + return wal_result_code::kOk; + }; + + ret->get_meta = [](const wal_object_type&, + const wal_object_type::log_type& log) -> wal_object_type::meta_result_type { + return wal_object_type::meta_result_type::make_success(log.timepoint, log.log_key, log.action); + }; + + ret->set_meta = [](const wal_object_type&, wal_object_type::log_type& log, const wal_object_type::meta_type& meta) { + log.action = meta.action_case; + log.timepoint = meta.timepoint; + log.log_key = meta.log_key; + }; + + ret->merge_log = [](const wal_object_type&, wal_object_type::callback_param_type, wal_object_type::log_type& to, + const wal_object_type::log_type& from) { + ++details::g_test_wal_object_stats.merge_count; + to.data = from.data; + }; + + ret->get_log_key = [](const wal_object_type&, const wal_object_type::log_type& log) -> wal_object_type::log_key_type { + return log.log_key; + }; + + ret->alloc_log_key = [](wal_object_type&, + wal_object_type::callback_param_type) -> wal_object_type::log_key_result_type { + return wal_object_type::log_key_result_type::make_success(++details::g_test_wal_object_stats.key_alloc); + }; + + ret->on_log_added = [](wal_object_type&, const wal_object_type::log_pointer&) { + ++details::g_test_wal_object_stats.event_on_log_added; + }; + + ret->on_log_removed = [](wal_object_type&, const wal_object_type::log_pointer&) { + ++details::g_test_wal_object_stats.event_on_log_removed; + }; + + ret->delegate_action[test_wal_object_log_action::kDoNothing] = + [](wal_object_type&, const wal_object_type::log_type& log, + wal_object_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.delegate_action_count; + details::g_test_wal_object_stats.last_log = log; + + return wal_result_code::kOk; + }; + + ret->delegate_action[test_wal_object_log_action::kRecursivePushBack] = + [](wal_object_type& wal, const wal_object_type::log_type& log, + wal_object_type::callback_param_type param) -> wal_result_code { + ++details::g_test_wal_object_stats.delegate_action_count; + details::g_test_wal_object_stats.last_log = log; + + auto new_log = wal.allocate_log(log.timepoint, test_wal_object_log_action::kDoNothing, param); + new_log->data = log.data + 1; + CASE_EXPECT_TRUE(wal_result_code::kPending == wal.emplace_back(std::move(new_log), param)); + + return wal_result_code::kOk; + }; + + ret->default_action = [](wal_object_type&, const wal_object_type::log_type& log, + wal_object_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.default_action_count; + details::g_test_wal_object_stats.last_log = log; + + return wal_result_code::kOk; + }; + + return ret; +} + +static test_wal_object_type::congfigure_pointer create_configure() { + test_wal_object_type::congfigure_pointer ret = std::make_shared(); + test_wal_object_type::default_configure(*ret); + + ret->gc_expire_duration = std::chrono::duration_cast(std::chrono::seconds{8}); + ret->max_log_size = 8; + ret->gc_log_size = 4; + + return ret; +} + +CASE_TEST(wal_object, create_failed) { + test_wal_object_log_storage_type storage; + // test_wal_object_context ctx; + + auto conf = create_configure(); + auto vtable = create_vtable(); + CASE_EXPECT_EQ(nullptr, test_wal_object_type::create(vtable, nullptr, &storage)); + CASE_EXPECT_EQ(nullptr, test_wal_object_type::create(nullptr, conf, &storage)); + + auto vtable_1 = vtable; + vtable_1->get_meta = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_object_type::create(vtable_1, conf, &storage)); + + auto vtable_2 = vtable; + vtable_2->get_log_key = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_object_type::create(vtable_2, conf, &storage)); + + auto vtable_3 = vtable; + vtable_3->alloc_log_key = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_object_type::create(vtable_3, conf, &storage)); +} + +CASE_TEST(wal_object, load_and_dump) { + auto old_action_count = + details::g_test_wal_object_stats.default_action_count + details::g_test_wal_object_stats.delegate_action_count; + + test_wal_object_log_storage_type load_storege; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + load_storege.global_ignore = 123; + load_storege.logs.push_back(test_wal_object_log_type{now, 124, test_wal_object_log_action::kDoNothing, 124}); + load_storege.logs.push_back(test_wal_object_log_type{now, 125, test_wal_object_log_action::kFallbackDefault, 125}); + load_storege.logs.push_back(test_wal_object_log_type{now, 126, test_wal_object_log_action::kRecursivePushBack, 126}); + + test_wal_object_log_storage_type storage; + test_wal_object_context ctx; + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto wal_obj = test_wal_object_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!wal_obj); + if (!wal_obj) { + return; + } + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kOk == wal_obj->load(load_storege, ctx)); + CASE_EXPECT_EQ(old_action_count, details::g_test_wal_object_stats.default_action_count + + details::g_test_wal_object_stats.delegate_action_count); + + CASE_EXPECT_EQ(123, storage.global_ignore); + CASE_EXPECT_EQ(3, storage.logs.size()); + CASE_EXPECT_EQ(124, storage.logs[0].data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == storage.logs[2].action); + + // dump + test_wal_object_log_storage_type dump_storege; + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kOk == wal_obj->dump(dump_storege, ctx)); + + CASE_EXPECT_EQ(123, dump_storege.global_ignore); + CASE_EXPECT_EQ(3, dump_storege.logs.size()); + CASE_EXPECT_EQ(124, dump_storege.logs[0].data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == dump_storege.logs[2].action); +} + +CASE_TEST(wal_object, add_action) { + test_wal_object_log_storage_type storage; + test_wal_object_context ctx; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto wal_obj = test_wal_object_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!wal_obj); + if (!wal_obj) { + return; + } + + do { + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kDoNothing == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(1, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 1, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + int64_t find_key = 0; + do { + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kRecursivePushBack, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + find_key = log->log_key; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(3, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 2, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + do { + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kFallbackDefault, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kFallbackDefault == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(4, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_default_action_count + 1, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + // ============ iterators ============ + { + auto find_ptr = wal_obj->find_log(find_key); + CASE_EXPECT_TRUE(!!find_ptr); + CASE_EXPECT_EQ(find_key + 100, find_ptr->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == find_ptr->action); + } + + do { + auto range = wal_obj->log_all_range(); + CASE_EXPECT_TRUE(range.first != wal_obj->log_end()); + if (range.first == wal_obj->log_end()) { + break; + } + + CASE_EXPECT_TRUE(range.first == wal_obj->log_begin()); + CASE_EXPECT_TRUE(range.second == wal_obj->log_end()); + + CASE_EXPECT_EQ(find_key + 99, (*range.first)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kDoNothing == (*range.first)->action); + } while (false); + + do { + const test_wal_object_type& obj = *wal_obj; + auto range = obj.log_all_range(); + CASE_EXPECT_TRUE(range.first != wal_obj->log_end()); + if (range.first == wal_obj->log_end()) { + break; + } + + CASE_EXPECT_TRUE(range.first == obj.log_cbegin()); + CASE_EXPECT_TRUE(range.second == obj.log_cend()); + + CASE_EXPECT_EQ(find_key + 99, (*range.first)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kDoNothing == (*range.first)->action); + } while (false); + + do { + auto iter = wal_obj->log_lower_bound(find_key); + CASE_EXPECT_TRUE(iter != wal_obj->log_end()); + if (iter == wal_obj->log_end()) { + break; + } + + CASE_EXPECT_EQ(find_key + 100, (*iter)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == (*iter)->action); + } while (false); + + do { + const test_wal_object_type& obj = *wal_obj; + auto iter = obj.log_lower_bound(find_key); + CASE_EXPECT_TRUE(iter != obj.log_cend()); + if (iter == obj.log_cend()) { + break; + } + + CASE_EXPECT_EQ(find_key + 100, (*iter)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == (*iter)->action); + } while (false); + + do { + auto iter = wal_obj->log_upper_bound(find_key + 1); + CASE_EXPECT_TRUE(iter != wal_obj->log_end()); + if (iter == wal_obj->log_end()) { + break; + } + + CASE_EXPECT_EQ(find_key + 102, (*iter)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kFallbackDefault == (*iter)->action); + } while (false); + + do { + const test_wal_object_type& obj = *wal_obj; + auto iter = obj.log_upper_bound(find_key + 1); + CASE_EXPECT_TRUE(iter != obj.log_cend()); + if (iter == obj.log_cend()) { + break; + } + + CASE_EXPECT_EQ(find_key + 102, (*iter)->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kFallbackDefault == (*iter)->action); + } while (false); +} + +CASE_TEST(wal_object, gc) { + test_wal_object_log_storage_type storage; + test_wal_object_context ctx; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + + auto conf = create_configure(); + auto vtable = create_vtable(); + + conf->gc_expire_duration = std::chrono::duration_cast(std::chrono::seconds{8}); + conf->max_log_size = 8; + conf->gc_log_size = 4; + + auto wal_obj = test_wal_object_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!wal_obj); + if (!wal_obj) { + return; + } + + CASE_EXPECT_EQ(4, wal_obj->get_configure().gc_log_size); + CASE_EXPECT_EQ(8, wal_obj->get_configure().max_log_size); + CASE_EXPECT_TRUE(std::chrono::duration_cast(std::chrono::seconds{8}) == + wal_obj->get_configure().gc_expire_duration); + auto old_remove_count = details::g_test_wal_object_stats.event_on_log_removed; + auto old_add_count = details::g_test_wal_object_stats.event_on_log_added; + + auto begin_key = details::g_test_wal_object_stats.key_alloc; + + for (int i = 0; i < 3; ++i) { + do { + now += std::chrono::duration_cast(std::chrono::seconds{1}); + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kDoNothing == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 1, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + do { + now += std::chrono::duration_cast(std::chrono::seconds{1}); + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kRecursivePushBack, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 2, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + do { + now += std::chrono::duration_cast(std::chrono::seconds{1}); + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_object_stats.key_alloc; + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kFallbackDefault, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_object_log_action::kFallbackDefault == log->action); + CASE_EXPECT_TRUE(now == log->timepoint); + + wal_obj->push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count + 1, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + } + + CASE_EXPECT_EQ(4, wal_obj->gc(now)); + CASE_EXPECT_EQ(8, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_add_count + 12, details::g_test_wal_object_stats.event_on_log_added); + CASE_EXPECT_EQ(old_remove_count + 4, details::g_test_wal_object_stats.event_on_log_removed); + + now += std::chrono::duration_cast(std::chrono::seconds{4}); + CASE_EXPECT_EQ(0, wal_obj->gc(now, &begin_key, 1)); + CASE_EXPECT_EQ(8, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_remove_count + 4, details::g_test_wal_object_stats.event_on_log_removed); + + CASE_EXPECT_EQ(1, wal_obj->gc(now, nullptr, 1)); + CASE_EXPECT_EQ(7, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_remove_count + 5, details::g_test_wal_object_stats.event_on_log_removed); + + CASE_EXPECT_EQ(2, wal_obj->gc(now, nullptr)); + CASE_EXPECT_EQ(5, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_remove_count + 7, details::g_test_wal_object_stats.event_on_log_removed); + + now += std::chrono::duration_cast(std::chrono::seconds{2}); + CASE_EXPECT_EQ(1, wal_obj->gc(now, nullptr)); + CASE_EXPECT_EQ(4, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_remove_count + 8, details::g_test_wal_object_stats.event_on_log_removed); +} diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp new file mode 100644 index 00000000..7044d82b --- /dev/null +++ b/test/case/wal_publisher_test.cpp @@ -0,0 +1,279 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "frame/test_macros.h" + +enum class test_wal_publisher_log_action { + kDoNothing = 0, + kRecursivePushBack, + kFallbackDefault, +}; + +struct test_wal_publisher_log_type { + util::distributed_system::wal_time_point timepoint; + int64_t log_key; + test_wal_publisher_log_action action; + int data; +}; + +struct test_wal_publisher_storage_type { + std::vector logs; + int64_t global_ignore; +}; + +struct test_wal_publisher_log_action_getter { + test_wal_publisher_log_action operator()(const test_wal_publisher_log_type& log) { return log.action; } +}; + +struct test_wal_publisher_context {}; + +struct test_wal_publisher_private_type { + test_wal_publisher_storage_type* storage; +}; + +using test_wal_publisher_log_operator = + util::distributed_system::wal_log_operator; + +using test_wal_publisher_subscriber_type = + util::distributed_system::wal_subscriber; + +using test_wal_publisher_type = + util::distributed_system::wal_publisher; + +struct test_wal_publisher_stats { + int64_t key_alloc; + size_t merge_count; + size_t delegate_action_count; + size_t default_action_count; + size_t event_on_log_added; + size_t event_on_log_removed; + + size_t send_subscribe_response; + size_t event_on_subscribe_added; + size_t event_on_subscribe_removed; + + size_t last_event_subscriber_count; + size_t last_event_log_count; + + test_wal_publisher_type::object_type::log_type last_log; + test_wal_publisher_type::subscriber_pointer last_subscriber; +}; + +namespace details { +test_wal_publisher_stats g_test_wal_object_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +} + +static test_wal_publisher_type::vtable_pointer create_vtable() { + using wal_object_type = test_wal_publisher_type::object_type; + using wal_publisher_type = test_wal_publisher_type; + using wal_result_code = util::distributed_system::wal_result_code; + + wal_publisher_type::vtable_pointer ret = std::make_shared(); + + // callbacks for wal_object + ret->load = [](wal_object_type& wal, const wal_object_type::storage_type& from, + wal_object_type::callback_param_type) -> wal_result_code { + *wal.get_private_data().storage = from; + return wal_result_code::kOk; + }; + + ret->dump = [](const wal_object_type& wal, wal_object_type::storage_type& to, + wal_object_type::callback_param_type) -> wal_result_code { + to = *wal.get_private_data().storage; + return wal_result_code::kOk; + }; + + ret->get_meta = [](const wal_object_type&, + const wal_object_type::log_type& log) -> wal_object_type::meta_result_type { + return wal_object_type::meta_result_type::make_success(log.timepoint, log.log_key, log.action); + }; + + ret->set_meta = [](const wal_object_type&, wal_object_type::log_type& log, const wal_object_type::meta_type& meta) { + log.action = meta.action_case; + log.timepoint = meta.timepoint; + log.log_key = meta.log_key; + }; + + ret->merge_log = [](const wal_object_type&, wal_object_type::callback_param_type, wal_object_type::log_type& to, + const wal_object_type::log_type& from) { + ++details::g_test_wal_object_stats.merge_count; + to.data = from.data; + }; + + ret->get_log_key = [](const wal_object_type&, const wal_object_type::log_type& log) -> wal_object_type::log_key_type { + return log.log_key; + }; + + ret->alloc_log_key = [](wal_object_type&, + wal_object_type::callback_param_type) -> wal_object_type::log_key_result_type { + return wal_object_type::log_key_result_type::make_success(++details::g_test_wal_object_stats.key_alloc); + }; + + ret->on_log_added = [](wal_object_type&, const wal_object_type::log_pointer&) { + ++details::g_test_wal_object_stats.event_on_log_added; + }; + + ret->on_log_removed = [](wal_object_type&, const wal_object_type::log_pointer&) { + ++details::g_test_wal_object_stats.event_on_log_removed; + }; + + ret->delegate_action[test_wal_publisher_log_action::kDoNothing] = + [](wal_object_type&, const wal_object_type::log_type& log, + wal_object_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.delegate_action_count; + details::g_test_wal_object_stats.last_log = log; + details::g_test_wal_object_stats.last_event_subscriber_count = 0; + details::g_test_wal_object_stats.last_event_log_count = 1; + + return wal_result_code::kOk; + }; + + ret->delegate_action[test_wal_publisher_log_action::kRecursivePushBack] = + [](wal_object_type& wal, const wal_object_type::log_type& log, + wal_object_type::callback_param_type param) -> wal_result_code { + ++details::g_test_wal_object_stats.delegate_action_count; + details::g_test_wal_object_stats.last_log = log; + details::g_test_wal_object_stats.last_event_subscriber_count = 0; + details::g_test_wal_object_stats.last_event_log_count = 1; + + auto new_log = wal.allocate_log(log.timepoint, test_wal_publisher_log_action::kDoNothing, param); + new_log->data = log.data + 1; + wal.emplace_back(std::move(new_log), param); + + return wal_result_code::kOk; + }; + + ret->default_action = [](wal_object_type&, const wal_object_type::log_type& log, + wal_object_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.default_action_count; + details::g_test_wal_object_stats.last_log = log; + details::g_test_wal_object_stats.last_event_subscriber_count = 0; + details::g_test_wal_object_stats.last_event_log_count = 1; + + return wal_result_code::kOk; + }; + + // ============ callbacks for wal_publisher ============ + ret->send_snapshot = [](wal_publisher_type& publisher, wal_publisher_type::subscriber_iterator begin, + wal_publisher_type::subscriber_iterator end, + wal_publisher_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.default_action_count; + + details::g_test_wal_object_stats.last_event_subscriber_count = 0; + while (begin != end) { + ++begin; + ++details::g_test_wal_object_stats.last_event_subscriber_count; + } + details::g_test_wal_object_stats.last_event_log_count = publisher.get_private_data().storage->logs.size(); + if (!publisher.get_private_data().storage->logs.empty()) { + details::g_test_wal_object_stats.last_log = *publisher.get_private_data().storage->logs.rbegin(); + } + + return wal_result_code::kOk; + }; + + ret->send_logs = [](wal_publisher_type&, wal_publisher_type::log_const_iterator log_begin, + wal_publisher_type::log_const_iterator log_end, + wal_publisher_type::subscriber_iterator subscriber_begin, + wal_publisher_type::subscriber_iterator subscriber_end, + wal_publisher_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.default_action_count; + + details::g_test_wal_object_stats.last_event_subscriber_count = 0; + while (subscriber_begin != subscriber_end) { + details::g_test_wal_object_stats.last_subscriber = subscriber_begin->second; + ++subscriber_begin; + ++details::g_test_wal_object_stats.last_event_subscriber_count; + } + + details::g_test_wal_object_stats.last_event_log_count = 0; + while (log_begin != log_end) { + details::g_test_wal_object_stats.last_log = **log_begin; + ++log_begin; + ++details::g_test_wal_object_stats.last_event_log_count; + } + + return wal_result_code::kOk; + }; + + ret->subscribe_response = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer&, wal_result_code, + wal_publisher_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.send_subscribe_response; + return wal_result_code::kOk; + }; + + ret->check_subscriber = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, + wal_publisher_type::callback_param_type) -> bool { + details::g_test_wal_object_stats.last_subscriber = subscriber; + return true; + }; + + ret->on_subscriber_added = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, + wal_publisher_type::callback_param_type) { + ++details::g_test_wal_object_stats.event_on_subscribe_added; + details::g_test_wal_object_stats.last_subscriber = subscriber; + + return wal_result_code::kOk; + }; + + ret->on_subscriber_removed = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, + util::distributed_system::wal_unsubscribe_reason, + wal_publisher_type::callback_param_type) { + ++details::g_test_wal_object_stats.event_on_subscribe_removed; + details::g_test_wal_object_stats.last_subscriber = subscriber; + + return wal_result_code::kOk; + }; + + return ret; +} + +static test_wal_publisher_type::congfigure_pointer create_configure() { + test_wal_publisher_type::congfigure_pointer ret = test_wal_publisher_type::make_configure(); + + ret->gc_expire_duration = std::chrono::duration_cast(std::chrono::seconds{8}); + ret->max_log_size = 8; + ret->gc_log_size = 4; + ret->subscriber_timeout = std::chrono::duration_cast(std::chrono::seconds{5}); + + return ret; +} + +CASE_TEST(wal_publisher, create_failed) { + test_wal_publisher_storage_type storage; + // test_wal_publisher_context ctx; + + auto conf = create_configure(); + auto vtable = create_vtable(); + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable, nullptr, &storage)); + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(nullptr, conf, &storage)); + + auto vtable_1 = vtable; + vtable_1->get_meta = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_1, conf, &storage)); + + auto vtable_2 = vtable; + vtable_2->get_log_key = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_2, conf, &storage)); + + auto vtable_3 = vtable; + vtable_3->alloc_log_key = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_3, conf, &storage)); + + auto vtable_4 = vtable; + vtable_4->send_snapshot = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_4, conf, &storage)); + + auto vtable_5 = vtable; + vtable_5->send_logs = nullptr; + CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_5, conf, &storage)); +} \ No newline at end of file From 8eefb84d2d5947d9450ffb016c4e35987a743209 Mon Sep 17 00:00:00 2001 From: owent Date: Sun, 15 Aug 2021 23:54:48 +0800 Subject: [PATCH 04/12] Fix ataptor for wal_object/wal_publisher --- include/design_pattern/result_type.h | 1 - include/distributed_system/wal_publisher.h | 7 +- include/distributed_system/wal_subscriber.h | 2 +- include/log/log_wrapper.h | 38 +++++------ test/case/wal_object_test.cpp | 2 +- test/case/wal_publisher_test.cpp | 72 ++++++++++----------- 6 files changed, 60 insertions(+), 62 deletions(-) diff --git a/include/design_pattern/result_type.h b/include/design_pattern/result_type.h index c65a4cea..dd744f7f 100644 --- a/include/design_pattern/result_type.h +++ b/include/design_pattern/result_type.h @@ -135,7 +135,6 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY static UTIL_FORCEINLINE void move_storage(storage_type &out, storage_type &&in) noexcept { out.first = in.first; - memset(&in.first, 0, sizeof(in.first)); } static UTIL_FORCEINLINE void swap(storage_type &l, storage_type &r) noexcept { diff --git a/include/distributed_system/wal_publisher.h b/include/distributed_system/wal_publisher.h index 8c5c0a3d..a974b2f2 100644 --- a/include/distributed_system/wal_publisher.h +++ b/include/distributed_system/wal_publisher.h @@ -116,8 +116,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { }; public: - template - explicit wal_publisher(construct_helper& helper, ArgsT&&... args) + explicit wal_publisher(construct_helper& helper) : vtable_(helper.vt), configure_(helper.conf), wal_object_(helper.wal_object), @@ -148,7 +147,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { return nullptr; } - return std::make_shared(helper, std::forward(args)...); + return std::make_shared(helper); } static congfigure_pointer make_configure() { @@ -184,7 +183,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { return nullptr; } - return wal_object_->allocate_log(now, action_case, param, std::forward(args)); + return wal_object_->allocate_log(now, action_case, param, std::forward(args)...); } wal_result_code emplace_back_log(log_pointer&& log, callback_param_type param) { diff --git a/include/distributed_system/wal_subscriber.h b/include/distributed_system/wal_subscriber.h index 7cf5f51d..cf87cf2e 100644 --- a/include/distributed_system/wal_subscriber.h +++ b/include/distributed_system/wal_subscriber.h @@ -275,7 +275,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_subscriber { inline const private_data_type& get_private_data() const noexcept { return private_data_; } inline private_data_type& get_private_data() noexcept { return private_data_; } - inline duration get_heartbeat_timeout() const noexcept { return duration; } + inline duration get_heartbeat_timeout() const noexcept { return heartbeat_timeout_; } inline void set_heartbeat_timeout(duration timeout) noexcept { heartbeat_timeout_ = timeout; } inline time_point get_last_heartbeat_time_point() const noexcept { return last_heartbeat_timepoint_; } diff --git a/include/log/log_wrapper.h b/include/log/log_wrapper.h index cb734b3f..3ae95221 100644 --- a/include/log/log_wrapper.h +++ b/include/log/log_wrapper.h @@ -41,11 +41,11 @@ LIBATFRAME_UTILS_API_HEAD_ONLY auto make_format_args(TARGS &&...args) { } template -LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(TFMT &&fmt, TARGS &&...args) { +LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(TFMT &&fmt_text, TARGS &&...args) { # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE format(std::forward(fmt), std::forward(args)...); + return LOG_WRAPPER_FWAPI_NAMESPACE format(std::forward(fmt_text), std::forward(args)...); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { return e.what(); @@ -58,11 +58,11 @@ LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(TFMT &&fmt, TARGS &&...args) { } template -LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt, TARGS &&...args) { +LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt_text, TARGS &&...args) { # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE format_to(out, std::forward(fmt), std::forward(args)...); + return LOG_WRAPPER_FWAPI_NAMESPACE format_to(out, std::forward(fmt_text), std::forward(args)...); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { const char *input_begin = e.what(); @@ -98,13 +98,13 @@ LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt, TARGS && template LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result format_to_n(OutputIt out, size_t n, - TFMT &&fmt, + TFMT &&fmt_text, TARGS &&...args) { # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif return LOG_WRAPPER_FWAPI_NAMESPACE format_to_n( - out, static_cast::size_type>(n), std::forward(fmt), + out, static_cast::size_type>(n), std::forward(fmt_text), std::forward(args)...); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { @@ -130,11 +130,11 @@ LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result -LIBATFRAME_UTILS_API_HEAD_ONLY std::string vformat(TFMT &&fmt, TARGS &&args) { +LIBATFRAME_UTILS_API_HEAD_ONLY std::string vformat(TFMT &&fmt_text, TARGS &&args) { # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE vformat(std::forward(fmt), std::forward(args)); + return LOG_WRAPPER_FWAPI_NAMESPACE vformat(std::forward(fmt_text), std::forward(args)); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { return e.what(); @@ -147,11 +147,11 @@ LIBATFRAME_UTILS_API_HEAD_ONLY std::string vformat(TFMT &&fmt, TARGS &&args) { } template -LIBATFRAME_UTILS_API_HEAD_ONLY OutputIt vformat_to(OutputIt out, TFMT &&fmt, TARGS &&args) { +LIBATFRAME_UTILS_API_HEAD_ONLY OutputIt vformat_to(OutputIt out, TFMT &&fmt_text, TARGS &&args) { # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE vformat_to(out, std::forward(fmt), std::forward(args)); + return LOG_WRAPPER_FWAPI_NAMESPACE vformat_to(out, std::forward(fmt_text), std::forward(args)); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { const char *input_begin = e.what(); @@ -237,18 +237,18 @@ class log_wrapper { LIBATFRAME_UTILS_API void log(const caller_info_t &caller, #ifdef _MSC_VER - _In_z_ _Printf_format_string_ const char *fmt, ...); + _In_z_ _Printf_format_string_ const char *fmt_text, ...); #elif (defined(__clang__) && __clang_major__ >= 3) - const char *fmt, ...) __attribute__((__format__(__printf__, 3, 4))); + const char *fmt_text, ...) __attribute__((__format__(__printf__, 3, 4))); #elif (defined(__GNUC__) && __GNUC__ >= 4) // 格式检查(成员函数有个隐含的this参数) # if defined(__MINGW32__) || defined(__MINGW64__) - const char *fmt, ...) __attribute__((format(__MINGW_PRINTF_FORMAT, 3, 4))); + const char *fmt_text, ...) __attribute__((format(__MINGW_PRINTF_FORMAT, 3, 4))); # else - const char *fmt, ...) __attribute__((format(printf, 3, 4))); + const char *fmt_text, ...) __attribute__((format(printf, 3, 4))); # endif #else - const char *fmt, ...); + const char *fmt_text, ...); #endif #if defined(LOG_WRAPPER_ENABLE_FWAPI) && LOG_WRAPPER_ENABLE_FWAPI @@ -561,23 +561,23 @@ class log_wrapper { // 控制台输出工具 #ifdef _MSC_VER -# define PSTDTERMCOLOR(os_ident, code, fmt, ...) \ +# define PSTDTERMCOLOR(os_ident, code, fmt_text, ...) \ \ { \ util::cli::shell_stream::shell_stream_opr log_wrapper_pstd_ss(&std::os_ident); \ log_wrapper_pstd_ss.open(code); \ log_wrapper_pstd_ss.close(); \ - printf(fmt, __VA_ARGS__); \ + printf(fmt_text, __VA_ARGS__); \ } #else -# define PSTDTERMCOLOR(os_ident, code, fmt, args...) \ +# define PSTDTERMCOLOR(os_ident, code, fmt_text, args...) \ \ { \ util::cli::shell_stream::shell_stream_opr log_wrapper_pstd_ss(&std::os_ident); \ log_wrapper_pstd_ss.open(code); \ log_wrapper_pstd_ss.close(); \ - printf(fmt, ##args); \ + printf(fmt_text, ##args); \ } #endif diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp index 12096a8e..fb2c42bb 100644 --- a/test/case/wal_object_test.cpp +++ b/test/case/wal_object_test.cpp @@ -55,7 +55,7 @@ struct test_wal_object_stats { }; namespace details { -test_wal_object_stats g_test_wal_object_stats{1, 0, 0, 0, 0, 0}; +test_wal_object_stats g_test_wal_object_stats{1, 0, 0, 0, 0, 0, test_wal_object_log_type()}; } static test_wal_object_type::vtable_pointer create_vtable() { diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp index 7044d82b..38793267 100644 --- a/test/case/wal_publisher_test.cpp +++ b/test/case/wal_publisher_test.cpp @@ -69,7 +69,7 @@ struct test_wal_publisher_stats { }; namespace details { -test_wal_publisher_stats g_test_wal_object_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; +test_wal_publisher_stats g_test_wal_publisher_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, test_wal_publisher_log_type(), nullptr}; } static test_wal_publisher_type::vtable_pointer create_vtable() { @@ -105,7 +105,7 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->merge_log = [](const wal_object_type&, wal_object_type::callback_param_type, wal_object_type::log_type& to, const wal_object_type::log_type& from) { - ++details::g_test_wal_object_stats.merge_count; + ++details::g_test_wal_publisher_stats.merge_count; to.data = from.data; }; @@ -115,24 +115,24 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->alloc_log_key = [](wal_object_type&, wal_object_type::callback_param_type) -> wal_object_type::log_key_result_type { - return wal_object_type::log_key_result_type::make_success(++details::g_test_wal_object_stats.key_alloc); + return wal_object_type::log_key_result_type::make_success(++details::g_test_wal_publisher_stats.key_alloc); }; ret->on_log_added = [](wal_object_type&, const wal_object_type::log_pointer&) { - ++details::g_test_wal_object_stats.event_on_log_added; + ++details::g_test_wal_publisher_stats.event_on_log_added; }; ret->on_log_removed = [](wal_object_type&, const wal_object_type::log_pointer&) { - ++details::g_test_wal_object_stats.event_on_log_removed; + ++details::g_test_wal_publisher_stats.event_on_log_removed; }; ret->delegate_action[test_wal_publisher_log_action::kDoNothing] = [](wal_object_type&, const wal_object_type::log_type& log, wal_object_type::callback_param_type) -> wal_result_code { - ++details::g_test_wal_object_stats.delegate_action_count; - details::g_test_wal_object_stats.last_log = log; - details::g_test_wal_object_stats.last_event_subscriber_count = 0; - details::g_test_wal_object_stats.last_event_log_count = 1; + ++details::g_test_wal_publisher_stats.delegate_action_count; + details::g_test_wal_publisher_stats.last_log = log; + details::g_test_wal_publisher_stats.last_event_subscriber_count = 0; + details::g_test_wal_publisher_stats.last_event_log_count = 1; return wal_result_code::kOk; }; @@ -140,10 +140,10 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->delegate_action[test_wal_publisher_log_action::kRecursivePushBack] = [](wal_object_type& wal, const wal_object_type::log_type& log, wal_object_type::callback_param_type param) -> wal_result_code { - ++details::g_test_wal_object_stats.delegate_action_count; - details::g_test_wal_object_stats.last_log = log; - details::g_test_wal_object_stats.last_event_subscriber_count = 0; - details::g_test_wal_object_stats.last_event_log_count = 1; + ++details::g_test_wal_publisher_stats.delegate_action_count; + details::g_test_wal_publisher_stats.last_log = log; + details::g_test_wal_publisher_stats.last_event_subscriber_count = 0; + details::g_test_wal_publisher_stats.last_event_log_count = 1; auto new_log = wal.allocate_log(log.timepoint, test_wal_publisher_log_action::kDoNothing, param); new_log->data = log.data + 1; @@ -154,10 +154,10 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->default_action = [](wal_object_type&, const wal_object_type::log_type& log, wal_object_type::callback_param_type) -> wal_result_code { - ++details::g_test_wal_object_stats.default_action_count; - details::g_test_wal_object_stats.last_log = log; - details::g_test_wal_object_stats.last_event_subscriber_count = 0; - details::g_test_wal_object_stats.last_event_log_count = 1; + ++details::g_test_wal_publisher_stats.default_action_count; + details::g_test_wal_publisher_stats.last_log = log; + details::g_test_wal_publisher_stats.last_event_subscriber_count = 0; + details::g_test_wal_publisher_stats.last_event_log_count = 1; return wal_result_code::kOk; }; @@ -166,16 +166,16 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->send_snapshot = [](wal_publisher_type& publisher, wal_publisher_type::subscriber_iterator begin, wal_publisher_type::subscriber_iterator end, wal_publisher_type::callback_param_type) -> wal_result_code { - ++details::g_test_wal_object_stats.default_action_count; + ++details::g_test_wal_publisher_stats.default_action_count; - details::g_test_wal_object_stats.last_event_subscriber_count = 0; + details::g_test_wal_publisher_stats.last_event_subscriber_count = 0; while (begin != end) { ++begin; - ++details::g_test_wal_object_stats.last_event_subscriber_count; + ++details::g_test_wal_publisher_stats.last_event_subscriber_count; } - details::g_test_wal_object_stats.last_event_log_count = publisher.get_private_data().storage->logs.size(); + details::g_test_wal_publisher_stats.last_event_log_count = publisher.get_private_data().storage->logs.size(); if (!publisher.get_private_data().storage->logs.empty()) { - details::g_test_wal_object_stats.last_log = *publisher.get_private_data().storage->logs.rbegin(); + details::g_test_wal_publisher_stats.last_log = *publisher.get_private_data().storage->logs.rbegin(); } return wal_result_code::kOk; @@ -186,20 +186,20 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { wal_publisher_type::subscriber_iterator subscriber_begin, wal_publisher_type::subscriber_iterator subscriber_end, wal_publisher_type::callback_param_type) -> wal_result_code { - ++details::g_test_wal_object_stats.default_action_count; + ++details::g_test_wal_publisher_stats.default_action_count; - details::g_test_wal_object_stats.last_event_subscriber_count = 0; + details::g_test_wal_publisher_stats.last_event_subscriber_count = 0; while (subscriber_begin != subscriber_end) { - details::g_test_wal_object_stats.last_subscriber = subscriber_begin->second; + details::g_test_wal_publisher_stats.last_subscriber = subscriber_begin->second; ++subscriber_begin; - ++details::g_test_wal_object_stats.last_event_subscriber_count; + ++details::g_test_wal_publisher_stats.last_event_subscriber_count; } - details::g_test_wal_object_stats.last_event_log_count = 0; + details::g_test_wal_publisher_stats.last_event_log_count = 0; while (log_begin != log_end) { - details::g_test_wal_object_stats.last_log = **log_begin; + details::g_test_wal_publisher_stats.last_log = **log_begin; ++log_begin; - ++details::g_test_wal_object_stats.last_event_log_count; + ++details::g_test_wal_publisher_stats.last_event_log_count; } return wal_result_code::kOk; @@ -207,20 +207,20 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->subscribe_response = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer&, wal_result_code, wal_publisher_type::callback_param_type) -> wal_result_code { - ++details::g_test_wal_object_stats.send_subscribe_response; + ++details::g_test_wal_publisher_stats.send_subscribe_response; return wal_result_code::kOk; }; ret->check_subscriber = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, wal_publisher_type::callback_param_type) -> bool { - details::g_test_wal_object_stats.last_subscriber = subscriber; + details::g_test_wal_publisher_stats.last_subscriber = subscriber; return true; }; ret->on_subscriber_added = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, wal_publisher_type::callback_param_type) { - ++details::g_test_wal_object_stats.event_on_subscribe_added; - details::g_test_wal_object_stats.last_subscriber = subscriber; + ++details::g_test_wal_publisher_stats.event_on_subscribe_added; + details::g_test_wal_publisher_stats.last_subscriber = subscriber; return wal_result_code::kOk; }; @@ -228,8 +228,8 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->on_subscriber_removed = [](wal_publisher_type&, const wal_publisher_type::subscriber_pointer& subscriber, util::distributed_system::wal_unsubscribe_reason, wal_publisher_type::callback_param_type) { - ++details::g_test_wal_object_stats.event_on_subscribe_removed; - details::g_test_wal_object_stats.last_subscriber = subscriber; + ++details::g_test_wal_publisher_stats.event_on_subscribe_removed; + details::g_test_wal_publisher_stats.last_subscriber = subscriber; return wal_result_code::kOk; }; @@ -276,4 +276,4 @@ CASE_TEST(wal_publisher, create_failed) { auto vtable_5 = vtable; vtable_5->send_logs = nullptr; CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_5, conf, &storage)); -} \ No newline at end of file +} From d07f3e7e529f3cd3af2f81e7a4debbfe788103cb Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 16 Aug 2021 00:33:40 +0800 Subject: [PATCH 05/12] Fix compactitable for fmt.dev. Fix unit test for result_type Signed-off-by: owent --- CMakeLists.txt | 2 +- include/log/log_formatter.h | 3 +++ include/log/log_wrapper.h | 37 +++++++++++++++++++++++++++++++--- test/case/result_type_test.cpp | 8 +++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 78fab950..5f2e9ec9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ enable_testing() project( atframe_utils - VERSION "2.3.1" + VERSION "2.3.2" HOMEPAGE_URL "https://github.com/atframework/atframe_utils" LANGUAGES C CXX) diff --git a/include/log/log_formatter.h b/include/log/log_formatter.h index c8d61571..542df78f 100644 --- a/include/log/log_formatter.h +++ b/include/log/log_formatter.h @@ -38,6 +38,9 @@ # ifndef LOG_WRAPPER_FWAPI_FMT_STRING # define LOG_WRAPPER_FWAPI_FMT_STRING(S) FMT_STRING(S) # endif +# if FMT_VERSION >= 80000 +# define LOG_WRAPPER_FWAPI_USING_FORMAT_STRING(...) fmt::format_string<__VA_ARGS__> +# endif #endif namespace util { diff --git a/include/log/log_wrapper.h b/include/log/log_wrapper.h index 3ae95221..4ade543e 100644 --- a/include/log/log_wrapper.h +++ b/include/log/log_wrapper.h @@ -40,12 +40,22 @@ LIBATFRAME_UTILS_API_HEAD_ONLY auto make_format_args(TARGS &&...args) { return LOG_WRAPPER_FWAPI_NAMESPACE make_format_args(std::forward(args)...); } +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING +template +LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(LOG_WRAPPER_FWAPI_USING_FORMAT_STRING(TARGS...) fmt_text, + TARGS &&...args) { +# else template LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(TFMT &&fmt_text, TARGS &&...args) { +# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE format(std::forward(fmt_text), std::forward(args)...); +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING + return LOG_WRAPPER_FWAPI_NAMESPACE format(fmt_text, std::forward(args)...); +# else + return LOG_WRAPPER_FWAPI_NAMESPACE format(std::forward(fmt_text), std::forward(args)...); +# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { return e.what(); @@ -57,12 +67,22 @@ LIBATFRAME_UTILS_API_HEAD_ONLY std::string format(TFMT &&fmt_text, TARGS &&...ar # endif } +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING +template +LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, LOG_WRAPPER_FWAPI_USING_FORMAT_STRING(TARGS...) fmt_text, + TARGS &&...args) { +# else template LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt_text, TARGS &&...args) { +# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE format_to(out, std::forward(fmt_text), std::forward(args)...); +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING + return LOG_WRAPPER_FWAPI_NAMESPACE format_to(out, fmt_text, std::forward(args)...); +# else + return LOG_WRAPPER_FWAPI_NAMESPACE format_to(out, std::forward(fmt_text), std::forward(args)...); +# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { const char *input_begin = e.what(); @@ -95,16 +115,27 @@ LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt_text, TAR # endif } +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING +template +LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result format_to_n( + OutputIt out, size_t n, LOG_WRAPPER_FWAPI_USING_FORMAT_STRING(TARGS...) fmt_text, TARGS &&...args) { +# else template LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result format_to_n(OutputIt out, size_t n, TFMT &&fmt_text, TARGS &&...args) { +# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif return LOG_WRAPPER_FWAPI_NAMESPACE format_to_n( - out, static_cast::size_type>(n), std::forward(fmt_text), + out, static_cast::size_type>(n), +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING + fmt_text, +# else + std::forward(fmt_text), +# endif std::forward(args)...); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { diff --git a/test/case/result_type_test.cpp b/test/case/result_type_test.cpp index 4be2601a..f3135e65 100644 --- a/test/case/result_type_test.cpp +++ b/test/case/result_type_test.cpp @@ -15,9 +15,15 @@ CASE_TEST(result_type, all_triviall) { auto success_obj = test_type::make_success(123); auto error_obj = test_type::make_error(456U); static_assert( - sizeof(std::aligned_storage::type) + sizeof(int32_t) >= sizeof(test_type), + sizeof(std::aligned_storage::type) + sizeof(int32_t) >= + sizeof(test_type), "size invalid for trivial result_type"); + CASE_MSG_INFO() << "sizeof(test_type::error_storage_type): " << sizeof(test_type::error_storage_type::storage_type) + << std::endl; + CASE_MSG_INFO() << "sizeof(std::aligned_storage::type): " + << sizeof(std::aligned_storage::type) + << std::endl; CASE_MSG_INFO() << "sizeof(result_type): " << sizeof(test_type) << std::endl; CASE_EXPECT_TRUE(success_obj.is_success()); CASE_EXPECT_FALSE(success_obj.is_error()); From ec7cf5e288888704310877c5788d4f938e899a32 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 16 Aug 2021 01:17:43 +0800 Subject: [PATCH 06/12] Fix compatitable for fmt.dev 8.0.1 --- include/log/log_wrapper.h | 35 ++++++++++++++++++++++------------- sample/sample.cpp | 6 ++++++ 2 files changed, 28 insertions(+), 13 deletions(-) diff --git a/include/log/log_wrapper.h b/include/log/log_wrapper.h index 4ade543e..3e0d538d 100644 --- a/include/log/log_wrapper.h +++ b/include/log/log_wrapper.h @@ -115,28 +115,23 @@ LIBATFRAME_UTILS_API_HEAD_ONLY auto format_to(OutputIt out, TFMT &&fmt_text, TAR # endif } -# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING -template -LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result format_to_n( - OutputIt out, size_t n, LOG_WRAPPER_FWAPI_USING_FORMAT_STRING(TARGS...) fmt_text, TARGS &&...args) { -# else template LIBATFRAME_UTILS_API_HEAD_ONLY LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result format_to_n(OutputIt out, size_t n, TFMT &&fmt_text, TARGS &&...args) { -# endif # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION try { # endif - return LOG_WRAPPER_FWAPI_NAMESPACE format_to_n( - out, static_cast::size_type>(n), # ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING - fmt_text, + return LOG_WRAPPER_FWAPI_NAMESPACE vformat_to_n( + out, static_cast::size_type>(n), std::forward(fmt_text), + LOG_WRAPPER_FWAPI_NAMESPACE make_format_args(std::forward(args)...)); # else - std::forward(fmt_text), + return LOG_WRAPPER_FWAPI_NAMESPACE format_to_n( + out, static_cast::size_type>(n), std::forward(fmt_text), + std::forward(args)...); # endif - std::forward(args)...); # if defined(LIBATFRAME_UTILS_ENABLE_EXCEPTION) && LIBATFRAME_UTILS_ENABLE_EXCEPTION } catch (const LOG_WRAPPER_FWAPI_NAMESPACE format_error &e) { const char *input_begin = e.what(); @@ -283,8 +278,13 @@ class log_wrapper { #endif #if defined(LOG_WRAPPER_ENABLE_FWAPI) && LOG_WRAPPER_ENABLE_FWAPI +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING + template + LIBATFRAME_UTILS_API_HEAD_ONLY void format_log(const caller_info_t &caller, FMT &&fmt_text, TARGS &&...args) { +# else template LIBATFRAME_UTILS_API_HEAD_ONLY void format_log(const caller_info_t &caller, TARGS &&...args) { +# endif log_operation_t writer; start_log(caller, writer); if (!log_sinks_.empty()) { @@ -292,8 +292,16 @@ class log_wrapper { try { # endif LOG_WRAPPER_FWAPI_NAMESPACE format_to_n_result result = - format_to_n(writer.buffer + writer.writen_size, writer.total_size - writer.writen_size - 1, - std::forward(args)...); +# ifdef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING + LOG_WRAPPER_FWAPI_NAMESPACE vformat_to_n( + writer.buffer + writer.writen_size, writer.total_size - writer.writen_size - 1, + std::forward(fmt_text), + LOG_WRAPPER_FWAPI_NAMESPACE make_format_args(std::forward(args)...)); +# else + LOG_WRAPPER_FWAPI_NAMESPACE format_to_n(writer.buffer + writer.writen_size, + writer.total_size - writer.writen_size - 1, + std::forward(args)...); +# endif if (result.size > 0) { writer.writen_size += static_cast(result.size); } @@ -651,3 +659,4 @@ LOG_WRAPPER_FWAPI_FORMAT_AS(typename ::util::log::log_wrapper::options_t::type, #include "config/compiler/template_suffix.h" #endif // _UTIL_LOG_LOG_WRAPPER_H_ + diff --git a/sample/sample.cpp b/sample/sample.cpp index 26d8e13b..8b429b4c 100644 --- a/sample/sample.cpp +++ b/sample/sample.cpp @@ -268,13 +268,17 @@ void log_sample_func7() { custom_obj.x = 123; custom_obj.y = __FUNCTION__; std::cout << "---------------- catch exception of log format APIs --------------" << std::endl; +#ifndef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING std::cout << "format: " << util::log::format("{},{},{},{}", 1, 2, 3) << std::endl; +#endif std::cout << "format: " << util::log::format("{},{}", 1, test_exception_for_log_formatter(true, true)) << std::endl; std::cout << "format: " << util::log::format("{},{}", 1, test_exception_for_log_formatter(false, true)) << std::endl; char string_buffer[256] = {0}; +#ifndef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING util::log::format_to(string_buffer, "{},{},{},{}", 1, 2, 3); std::cout << "format_to: " << string_buffer << std::endl; +#endif memset(string_buffer, 0, sizeof(string_buffer)); util::log::format_to(string_buffer, "{},{}", 1, test_exception_for_log_formatter(true, true)); std::cout << "format_to: " << string_buffer << std::endl; @@ -283,8 +287,10 @@ void log_sample_func7() { std::cout << "format_to: " << string_buffer << std::endl; memset(string_buffer, 0, sizeof(string_buffer)); +#ifndef LOG_WRAPPER_FWAPI_USING_FORMAT_STRING util::log::format_to_n(string_buffer, 100, "{},{},{},{}", 1, 2, 3); std::cout << "format_to_n: " << string_buffer << std::endl; +#endif memset(string_buffer, 0, sizeof(string_buffer)); util::log::format_to_n(string_buffer, 100, "{},{}", 1, test_exception_for_log_formatter(true, true)); std::cout << "format_to_n: " << string_buffer << std::endl; From 0b6fdb119c9f2208c5dd1e126644980e340a02ad Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 16 Aug 2021 13:28:07 +0800 Subject: [PATCH 07/12] Fix compatitability for old gcc --- include/distributed_system/wal_common_defs.h | 2 +- include/distributed_system/wal_object.h | 48 +++++-- include/distributed_system/wal_publisher.h | 1 + include/distributed_system/wal_subscriber.h | 6 +- test/case/wal_object_test.cpp | 128 +++++++++++++++++++ test/case/wal_publisher_test.cpp | 14 ++ 6 files changed, 187 insertions(+), 12 deletions(-) diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h index 325e917d..6c11f227 100644 --- a/include/distributed_system/wal_common_defs.h +++ b/include/distributed_system/wal_common_defs.h @@ -42,7 +42,7 @@ enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_result_code : int32_t { kOk = 0, - kIgnored = 101, + kIgnore = 101, kPending = 102, kMerge = 103, }; diff --git a/include/distributed_system/wal_object.h b/include/distributed_system/wal_object.h index 2366ac8a..13f476f1 100644 --- a/include/distributed_system/wal_object.h +++ b/include/distributed_system/wal_object.h @@ -43,8 +43,9 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { using action_case_type = typename log_operator_type::action_case_type; using log_key_result_type = typename log_operator_type::log_key_result_type; - using log_iterator = typename std::deque::iterator; - using log_const_iterator = typename std::deque::const_iterator; + using log_container_type = std::deque; + using log_iterator = typename log_container_type::iterator; + using log_const_iterator = typename log_container_type::const_iterator; using callback_param_type = CallbackParamT; using time_point = wal_time_point; using duration = wal_duration; @@ -149,7 +150,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { } static void default_configure(congfigure_type& out) { - out.gc_expire_duration = std::chrono::duration_cast(std::chrono::days{7}); + out.gc_expire_duration = std::chrono::duration_cast(std::chrono::hours(7 * 24)); out.max_log_size = 512; out.gc_log_size = 128; } @@ -178,6 +179,37 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { return vtable_->dump(*this, storage, param); } + /** + * @brief clear and assign all logs + * @note this function is useful when loading data and will not trigger event and action callback + * + * @tparam IteratorT iterator type + * @param begin + * @param end + */ + template + void assign_logs(IteratorT&& begin, IteratorT&& end) { + logs_.clear(); + logs_.assign(std::forward(begin), std::forward(end)); + } + + /** + * @brief clear and assign all logs + * @note this function is useful when loading data and will not trigger event and action callback + * + * @tparam ContainerT container type + * @param source + */ + template + void assign_logs(const ContainerT& source) { + logs_.assign(source.begin(), source.end()); + } + + void assign_logs(log_container_type&& source) { + logs_.swap(source); + source.clear(); + } + template log_pointer allocate_log(time_point now, action_case_type action_case, callback_param_type param, ArgsT&&... args) { if (!configure_ || !vtable_ || !vtable_->alloc_log_key || !vtable_->set_meta) { @@ -221,8 +253,8 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { ret = pusk_back_inner(std::move(log), param); } else { log_key_type this_key = vtable_->get_log_key(*this, *log); - if (global_ingore_ && log_key_compare_(this_key, *global_ingore_)) { - ret = wal_result_code::kIgnored; + if (global_ingore_ && !log_key_compare_(*global_ingore_, this_key)) { + ret = wal_result_code::kIgnore; } else { ret = pusk_back_inner(std::move(log), param); } @@ -285,7 +317,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { * @return size_t Log count removed */ size_t gc(time_point now, const log_key_type* hold = nullptr, size_t max_count = std::numeric_limits::max()) { - duration gc_expire_duration = std::chrono::duration_cast(std::chrono::days{7}); + duration gc_expire_duration = std::chrono::duration_cast(std::chrono::hours(7 * 24)); size_t max_log_size = 512; size_t gc_log_size = 128; if (configure_) { @@ -357,7 +389,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { } } - inline const std::deque& get_all_logs() const noexcept { return logs_; } + inline const log_container_type& get_all_logs() const noexcept { return logs_; } inline const private_data_type& get_private_data() const noexcept { return private_data_; } inline private_data_type& get_private_data() noexcept { return private_data_; } @@ -629,7 +661,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { std::unique_ptr global_ingore_; // ignore all log lower than this key // logs(libstdc++ is 512Byte for each block and maintain block index just like std::vector) - std::deque logs_; + log_container_type logs_; std::list > pending_logs_; }; diff --git a/include/distributed_system/wal_publisher.h b/include/distributed_system/wal_publisher.h index a974b2f2..8ff2997c 100644 --- a/include/distributed_system/wal_publisher.h +++ b/include/distributed_system/wal_publisher.h @@ -37,6 +37,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { using action_case_type = typename log_operator_type::action_case_type; using log_key_result_type = typename log_operator_type::log_key_result_type; + using log_container_type = typename object_type::log_container_type; using log_iterator = typename object_type::log_iterator; using log_const_iterator = typename object_type::log_const_iterator; using callback_param_type = typename object_type::callback_param_type; diff --git a/include/distributed_system/wal_subscriber.h b/include/distributed_system/wal_subscriber.h index cf87cf2e..1cb82fce 100644 --- a/include/distributed_system/wal_subscriber.h +++ b/include/distributed_system/wal_subscriber.h @@ -55,7 +55,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_subscriber { UTIL_DESIGN_PATTERN_NOMOVABLE(manager); UTIL_DESIGN_PATTERN_NOCOPYABLE(manager); - void remove_subscriber_timer(std::list::iterator& out) { + void remove_subscriber_timer(typename std::list::iterator& out) { if (out == subscribers_timer_.end()) { return; } @@ -262,7 +262,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_subscriber { public: template wal_subscriber(construct_helper&, manager& owner, const key_type& key, const time_point& now, const duration& timeout, - std::list::iterator timer_iter, ArgsT&&... args) + typename std::list::iterator timer_iter, ArgsT&&... args) : owner_(&owner), key_(key), last_heartbeat_timepoint_(now), @@ -292,7 +292,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_subscriber { duration heartbeat_timeout_; private_data_type private_data_; - std::list::iterator timer_iter_; + typename std::list::iterator timer_iter_; }; } // namespace distributed_system diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp index fb2c42bb..deb113c6 100644 --- a/test/case/wal_object_test.cpp +++ b/test/case/wal_object_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -12,6 +13,7 @@ enum class test_wal_object_log_action { kDoNothing = 0, kRecursivePushBack, + kIgnore, kFallbackDefault, }; @@ -35,6 +37,9 @@ struct test_wal_object_context {}; struct test_wal_object_private_type { test_wal_object_log_storage_type* storage; + + inline test_wal_object_private_type(): storage(nullptr) {} + inline explicit test_wal_object_private_type(test_wal_object_log_storage_type* input): storage(input) {} }; using test_wal_object_log_operator = @@ -68,6 +73,16 @@ static test_wal_object_type::vtable_pointer create_vtable() { wal_object_type::callback_param_type) -> wal_result_code { *wal.get_private_data().storage = from; wal.set_global_ingore_key(from.global_ignore); + + std::vector container; + for (auto& log : from.logs) { + container.emplace_back(std::make_shared(log)); + } + + wal.assign_logs(container.begin(), container.end()); + if (!from.logs.empty()) { + wal.set_last_removed_key((*from.logs.rbegin()).log_key - 1); + } return wal_result_code::kOk; }; @@ -136,6 +151,15 @@ static test_wal_object_type::vtable_pointer create_vtable() { return wal_result_code::kOk; }; + ret->delegate_action[test_wal_object_log_action::kIgnore] = + [](wal_object_type&, const wal_object_type::log_type& log, + wal_object_type::callback_param_type) -> wal_result_code { + ++details::g_test_wal_object_stats.delegate_action_count; + details::g_test_wal_object_stats.last_log = log; + + return wal_result_code::kIgnore; + }; + ret->default_action = [](wal_object_type&, const wal_object_type::log_type& log, wal_object_type::callback_param_type) -> wal_result_code { ++details::g_test_wal_object_stats.default_action_count; @@ -210,6 +234,10 @@ CASE_TEST(wal_object, load_and_dump) { CASE_EXPECT_EQ(124, storage.logs[0].data); CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == storage.logs[2].action); + CASE_EXPECT_EQ(3, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(124, (*wal_obj->get_all_logs().begin())->data); + CASE_EXPECT_TRUE(test_wal_object_log_action::kRecursivePushBack == (*wal_obj->get_all_logs().rbegin())->action); + // dump test_wal_object_log_storage_type dump_storege; CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kOk == wal_obj->dump(dump_storege, ctx)); @@ -497,3 +525,103 @@ CASE_TEST(wal_object, gc) { CASE_EXPECT_EQ(4, wal_obj->get_all_logs().size()); CASE_EXPECT_EQ(old_remove_count + 8, details::g_test_wal_object_stats.event_on_log_removed); } + + +CASE_TEST(wal_object, ignore) { + test_wal_object_log_storage_type storage; + test_wal_object_context ctx; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto wal_obj = test_wal_object_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!wal_obj); + if (!wal_obj) { + return; + } + + do { + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + wal_obj->set_global_ingore_key(log->log_key); + auto push_back_result = wal_obj->push_back(log, ctx); + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kIgnore == push_back_result); + + CASE_EXPECT_EQ(0, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count, details::g_test_wal_object_stats.delegate_action_count); + } while (false); + + do { + auto old_default_action_count = details::g_test_wal_object_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_object_stats.delegate_action_count; + + auto log = wal_obj->allocate_log(now, test_wal_object_log_action::kIgnore, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + auto push_back_result = wal_obj->push_back(log, ctx); + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kIgnore == push_back_result); + + CASE_EXPECT_EQ(0, wal_obj->get_all_logs().size()); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_object_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 1, details::g_test_wal_object_stats.delegate_action_count); + } while (false); +} + + +CASE_TEST(wal_object, reorder) { + test_wal_object_log_storage_type storage; + test_wal_object_context ctx; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto wal_obj = test_wal_object_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!wal_obj); + if (!wal_obj) { + return; + } + + test_wal_object_type::log_pointer log1; + test_wal_object_type::log_pointer log2; + do { + auto log1 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log1); + if (!log1) { + break; + } + log1->data = log1->log_key + 100; + } while (false); + + do { + auto log2 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log2); + if (!log2) { + break; + } + log2->data = log2->log_key + 100; + } while (false); + + if (!log1 || !log2) { + return; + } + + wal_obj->push_back(log2, ctx); + wal_obj->push_back(log1, ctx); + + auto iter = wal_obj->log_begin(); + CASE_EXPECT_EQ(log1.get(), (*iter).get()); + + ++ iter; + CASE_EXPECT_EQ(log2.get(), (*iter).get()); +} diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp index 38793267..9df11a15 100644 --- a/test/case/wal_publisher_test.cpp +++ b/test/case/wal_publisher_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -35,6 +36,9 @@ struct test_wal_publisher_context {}; struct test_wal_publisher_private_type { test_wal_publisher_storage_type* storage; + + inline test_wal_publisher_private_type(): storage(nullptr) {} + inline explicit test_wal_publisher_private_type(test_wal_publisher_storage_type* input): storage(input) {} }; using test_wal_publisher_log_operator = @@ -83,6 +87,16 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ret->load = [](wal_object_type& wal, const wal_object_type::storage_type& from, wal_object_type::callback_param_type) -> wal_result_code { *wal.get_private_data().storage = from; + + wal_publisher_type::log_container_type container; + for (auto& log : from.logs) { + container.emplace_back(std::make_shared(log)); + } + + wal.assign_logs(container); + if (!from.logs.empty()) { + wal.set_last_removed_key((*from.logs.rbegin()).log_key - 1); + } return wal_result_code::kOk; }; From 51301da04c2ef4b4b07db76acc748c56d3ce991c Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 16 Aug 2021 14:15:49 +0800 Subject: [PATCH 08/12] Fix usage of result_of --- include/distributed_system/wal_common_defs.h | 2 +- test/case/wal_object_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h index 6c11f227..9955d297 100644 --- a/include/distributed_system/wal_common_defs.h +++ b/include/distributed_system/wal_common_defs.h @@ -59,7 +59,7 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_log_action_getter_trait { #if defined(__cplusplus) && __cplusplus >= 201703L using type = std::invoke_result_t; #else - using type = typename std::result_of::type; + using type = typename std::result_of::type; #endif }; diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp index deb113c6..3606b8de 100644 --- a/test/case/wal_object_test.cpp +++ b/test/case/wal_object_test.cpp @@ -595,7 +595,7 @@ CASE_TEST(wal_object, reorder) { test_wal_object_type::log_pointer log1; test_wal_object_type::log_pointer log2; do { - auto log1 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + log1 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); CASE_EXPECT_TRUE(!!log1); if (!log1) { break; @@ -604,7 +604,7 @@ CASE_TEST(wal_object, reorder) { } while (false); do { - auto log2 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); + log2 = wal_obj->allocate_log(now, test_wal_object_log_action::kDoNothing, ctx); CASE_EXPECT_TRUE(!!log2); if (!log2) { break; From 213d4f0856b4f57e293736b99b4cafb30f438b72 Mon Sep 17 00:00:00 2001 From: owent Date: Mon, 16 Aug 2021 16:03:29 +0800 Subject: [PATCH 09/12] Fix compatibility for gcc 4.8 --- include/distributed_system/wal_common_defs.h | 11 +++++++++-- test/case/wal_object_test.cpp | 19 +++++++++++++------ test/case/wal_publisher_test.cpp | 18 ++++++++++++++---- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h index 9955d297..61b895d8 100644 --- a/include/distributed_system/wal_common_defs.h +++ b/include/distributed_system/wal_common_defs.h @@ -29,9 +29,16 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_meta_type { wal_time_point timepoint; LogKeyT log_key; ActionCaseT action_case; + + inline wal_meta_type() = default; + template + inline wal_meta_type(ToTimepointT&& t, ToLogKeyT&& k, ToActionCaseT&& act) + : timepoint(std::forward(t)), + log_key(std::forward(k)), + action_case(std::forward(act)) {} }; -enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_result_code : int32_t { +enum class wal_result_code : int32_t { kSubscriberNotFound = -201, kInitlization = -105, @@ -47,7 +54,7 @@ enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_result_code : int32_t { kMerge = 103, }; -enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_unsubscribe_reason : int32_t { +enum class wal_unsubscribe_reason : int32_t { kNone = 0, kTimeout = 1, kClientRequest = 2, diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp index 3606b8de..32f811ea 100644 --- a/test/case/wal_object_test.cpp +++ b/test/case/wal_object_test.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include #include @@ -17,6 +17,15 @@ enum class test_wal_object_log_action { kFallbackDefault, }; +namespace std { +template <> +struct hash { + std::size_t operator()(test_wal_object_log_action const& s) const noexcept { + return std::hash{}(static_cast(s)); + } +}; +} // namespace std + struct test_wal_object_log_type { util::distributed_system::wal_time_point timepoint; int64_t log_key; @@ -38,8 +47,8 @@ struct test_wal_object_context {}; struct test_wal_object_private_type { test_wal_object_log_storage_type* storage; - inline test_wal_object_private_type(): storage(nullptr) {} - inline explicit test_wal_object_private_type(test_wal_object_log_storage_type* input): storage(input) {} + inline test_wal_object_private_type() : storage(nullptr) {} + inline explicit test_wal_object_private_type(test_wal_object_log_storage_type* input) : storage(input) {} }; using test_wal_object_log_operator = @@ -526,7 +535,6 @@ CASE_TEST(wal_object, gc) { CASE_EXPECT_EQ(old_remove_count + 8, details::g_test_wal_object_stats.event_on_log_removed); } - CASE_TEST(wal_object, ignore) { test_wal_object_log_storage_type storage; test_wal_object_context ctx; @@ -578,7 +586,6 @@ CASE_TEST(wal_object, ignore) { } while (false); } - CASE_TEST(wal_object, reorder) { test_wal_object_log_storage_type storage; test_wal_object_context ctx; @@ -622,6 +629,6 @@ CASE_TEST(wal_object, reorder) { auto iter = wal_obj->log_begin(); CASE_EXPECT_EQ(log1.get(), (*iter).get()); - ++ iter; + ++iter; CASE_EXPECT_EQ(log2.get(), (*iter).get()); } diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp index 9df11a15..5bc1dce0 100644 --- a/test/case/wal_publisher_test.cpp +++ b/test/case/wal_publisher_test.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include #include @@ -16,6 +16,15 @@ enum class test_wal_publisher_log_action { kFallbackDefault, }; +namespace std { +template <> +struct hash { + std::size_t operator()(test_wal_publisher_log_action const& s) const noexcept { + return std::hash{}(static_cast(s)); + } +}; +} // namespace std + struct test_wal_publisher_log_type { util::distributed_system::wal_time_point timepoint; int64_t log_key; @@ -37,8 +46,8 @@ struct test_wal_publisher_context {}; struct test_wal_publisher_private_type { test_wal_publisher_storage_type* storage; - inline test_wal_publisher_private_type(): storage(nullptr) {} - inline explicit test_wal_publisher_private_type(test_wal_publisher_storage_type* input): storage(input) {} + inline test_wal_publisher_private_type() : storage(nullptr) {} + inline explicit test_wal_publisher_private_type(test_wal_publisher_storage_type* input) : storage(input) {} }; using test_wal_publisher_log_operator = @@ -73,7 +82,8 @@ struct test_wal_publisher_stats { }; namespace details { -test_wal_publisher_stats g_test_wal_publisher_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, test_wal_publisher_log_type(), nullptr}; +test_wal_publisher_stats g_test_wal_publisher_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, test_wal_publisher_log_type(), + nullptr}; } static test_wal_publisher_type::vtable_pointer create_vtable() { From ad68142641ac0b1b100bff9632c455fe58e24a0e Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 17 Aug 2021 16:04:29 +0800 Subject: [PATCH 10/12] Add unit test for wal_publisher --- include/distributed_system/wal_common_defs.h | 6 + include/distributed_system/wal_object.h | 48 ++- include/distributed_system/wal_publisher.h | 56 ++- test/case/wal_object_test.cpp | 22 +- test/case/wal_publisher_test.cpp | 385 ++++++++++++++++++- 5 files changed, 487 insertions(+), 30 deletions(-) diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h index 9955d297..cae17ceb 100644 --- a/include/distributed_system/wal_common_defs.h +++ b/include/distributed_system/wal_common_defs.h @@ -29,6 +29,12 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_meta_type { wal_time_point timepoint; LogKeyT log_key; ActionCaseT action_case; + + inline wal_meta_type() = default; + template + inline wal_meta_type(ToTimepointT&& t, ToLogKeyT&& k, ToActionCaseT&& act): + timepoint(std::forward(t)), log_key(std::forward(k)), + action_case(std::forward(act)) {} }; enum class LIBATFRAME_UTILS_API_HEAD_ONLY wal_result_code : int32_t { diff --git a/include/distributed_system/wal_object.h b/include/distributed_system/wal_object.h index 13f476f1..3839e633 100644 --- a/include/distributed_system/wal_object.h +++ b/include/distributed_system/wal_object.h @@ -164,7 +164,11 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { return wal_result_code::kActionNotSet; } - return vtable_->load(*this, storage, param); + wal_result_code ret = vtable_->load(*this, storage, param); + if (wal_result_code::kOk == ret && internal_event_on_loaded_) { + internal_event_on_loaded_(*this, storage, param); + } + return ret; } wal_result_code dump(storage_type& storage, callback_param_type param) { @@ -176,38 +180,50 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { return wal_result_code::kActionNotSet; } - return vtable_->dump(*this, storage, param); + wal_result_code ret = vtable_->dump(*this, storage, param); + if (wal_result_code::kOk == ret && internal_event_on_dumped_) { + internal_event_on_dumped_(*this, storage, param); + } + return ret; } /** * @brief clear and assign all logs * @note this function is useful when loading data and will not trigger event and action callback - * + * * @tparam IteratorT iterator type * @param begin - * @param end + * @param end */ - template + template void assign_logs(IteratorT&& begin, IteratorT&& end) { logs_.clear(); logs_.assign(std::forward(begin), std::forward(end)); + + if (internal_event_on_assign_) { + internal_event_on_assign_(*this); + } } /** * @brief clear and assign all logs * @note this function is useful when loading data and will not trigger event and action callback - * + * * @tparam ContainerT container type * @param source */ - template + template void assign_logs(const ContainerT& source) { - logs_.assign(source.begin(), source.end()); + assign_logs(source.begin(), source.end()); } void assign_logs(log_container_type&& source) { logs_.swap(source); source.clear(); + + if (internal_event_on_assign_) { + internal_event_on_assign_(*this); + } } template @@ -649,6 +665,17 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { } } + private: + template + friend class wal_publisher; + using callback_log_event_on_assign_fn_t = std::function; + + void set_internal_event_on_assign_logs(callback_log_event_on_assign_fn_t fn) { internal_event_on_assign_ = fn; } + + void set_internal_event_on_loaded(callback_load_fn_t fn) { internal_event_on_loaded_ = fn; } + + void set_internal_event_on_dumped(callback_dump_fn_t fn) { internal_event_on_dumped_ = fn; } + private: bool in_log_action_callback_; vtable_pointer vtable_; @@ -663,6 +690,11 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_object { // logs(libstdc++ is 512Byte for each block and maintain block index just like std::vector) log_container_type logs_; std::list > pending_logs_; + + // internal events + callback_log_event_on_assign_fn_t internal_event_on_assign_; + callback_load_fn_t internal_event_on_loaded_; + callback_dump_fn_t internal_event_on_dumped_; }; } // namespace distributed_system diff --git a/include/distributed_system/wal_publisher.h b/include/distributed_system/wal_publisher.h index 8ff2997c..be861af8 100644 --- a/include/distributed_system/wal_publisher.h +++ b/include/distributed_system/wal_publisher.h @@ -121,7 +121,16 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { : vtable_(helper.vt), configure_(helper.conf), wal_object_(helper.wal_object), - subscriber_manager_(helper.subscriber_manager) {} + subscriber_manager_(helper.subscriber_manager) { + if (wal_object_) { + wal_object_->set_internal_event_on_assign_logs([this](object_type& wal) { + // reset broadcast + if (!wal.get_all_logs().empty() && this->vtable_ && this->vtable_->get_log_key) { + this->set_broadcast_key_bound(this->vtable_->get_log_key(wal, **wal.get_all_logs().rbegin())); + } + }); + } + } template static std::shared_ptr create(vtable_pointer vt, congfigure_pointer conf, ArgsT&&... args) { @@ -178,6 +187,14 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { return wal_object_->dump(storage, param); } + template + void assign_logs(ArgsT&&... args) { + if (!wal_object_) { + return; + } + wal_object_->assign_logs(std::forward(args)...); + } + template log_pointer allocate_log(time_point now, action_case_type action_case, callback_param_type param, ArgsT&&... args) { if (!wal_object_) { @@ -285,10 +302,10 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { template subscriber_pointer create_subscriber(const subscriber_key_type& key, const time_point& now, callback_param_type param, ArgsT&&... args) { - subscriber_pointer subscriber = find_subscriber(key); + subscriber_pointer subscriber = find_subscriber(key, param); if (subscriber) { subscriber->set_heartbeat_timeout(configure_->subscriber_timeout); - subscriber_manager_->subscribe(*subscriber, now); + subscriber_manager_->subscribe(subscriber, now); return subscriber; } @@ -303,19 +320,21 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { send_snapshot(range.first, range.second, std::move(param)); } } + + return subscriber; } void remove_subscriber(const subscriber_key_type& key, wal_unsubscribe_reason reason, callback_param_type param) { subscriber_pointer subscriber = subscriber_manager_->unsubscribe(key, reason); if (subscriber && vtable_ && vtable_->on_subscriber_removed) { - vtable_->on_subscriber_removed(this, subscriber, reason, param); + vtable_->on_subscriber_removed(*this, subscriber, reason, param); } } void remove_subscriber(const subscriber_pointer& checked, wal_unsubscribe_reason reason, callback_param_type param) { subscriber_pointer subscriber = subscriber_manager_->unsubscribe(checked, reason); if (subscriber && vtable_ && vtable_->on_subscriber_removed) { - vtable_->on_subscriber_removed(this, subscriber, reason, param); + vtable_->on_subscriber_removed(*this, subscriber, reason, param); } } @@ -334,7 +353,13 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { } // GC logs - size_t res = wal_object_->gc(now, &broadcast_key_bound_, round); + size_t res; + if (broadcast_key_bound_) { + res = wal_object_->gc(now, broadcast_key_bound_.get(), round);; + } else { + res = wal_object_->gc(now, nullptr, round); + } + if (res > 0) { has_event = true; ret += res; @@ -382,7 +407,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { } } - log_const_iterator log_iter = wal_object_->log_upper_bound(key); + log_const_iterator log_iter = wal_object_->log_upper_bound(last_checkpoint); if (log_iter != wal_object_->log_cend()) { auto iters = subscriber_manager_->find_iterator(key); auto notify_result = send_logs(log_iter, wal_object_->log_cend(), iters.first, iters.second, std::move(param)); @@ -392,6 +417,17 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { return send_subscribe_response(subscriber, wal_result_code::kOk, param); } + template + void set_broadcast_key_bound(ArgsT&&... args) { + if (broadcast_key_bound_) { + *broadcast_key_bound_ = log_key_type{std::forward(args)...}; + } else { + broadcast_key_bound_.reset(new log_key_type{std::forward(args)...}); + } + } + + const std::unique_ptr& get_broadcast_key_bound() const { return broadcast_key_bound_; } + template size_t broadcast(ParamT&& param) { if (!vtable_ || !vtable_->get_log_key) { @@ -423,11 +459,7 @@ class LIBATFRAME_UTILS_API_HEAD_ONLY wal_publisher { ++ret; } - if (broadcast_key_bound_) { - *broadcast_key_bound_ = vtable_->get_log_key(*wal_object_, **last); - } else { - broadcast_key_bound_.reset(new log_key_type{vtable_->get_log_key(*wal_object_, **last)}); - } + set_broadcast_key_bound(vtable_->get_log_key(*wal_object_, **last)); return ret; } diff --git a/test/case/wal_object_test.cpp b/test/case/wal_object_test.cpp index 3606b8de..d6708e28 100644 --- a/test/case/wal_object_test.cpp +++ b/test/case/wal_object_test.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include #include @@ -17,6 +17,15 @@ enum class test_wal_object_log_action { kFallbackDefault, }; +namespace std { +template <> +struct hash { + std::size_t operator()(test_wal_object_log_action const& s) const noexcept { + return std::hash{}(static_cast(s)); + } +}; +} // namespace std + struct test_wal_object_log_type { util::distributed_system::wal_time_point timepoint; int64_t log_key; @@ -38,8 +47,8 @@ struct test_wal_object_context {}; struct test_wal_object_private_type { test_wal_object_log_storage_type* storage; - inline test_wal_object_private_type(): storage(nullptr) {} - inline explicit test_wal_object_private_type(test_wal_object_log_storage_type* input): storage(input) {} + inline test_wal_object_private_type() : storage(nullptr) {} + inline explicit test_wal_object_private_type(test_wal_object_log_storage_type* input) : storage(input) {} }; using test_wal_object_log_operator = @@ -524,8 +533,10 @@ CASE_TEST(wal_object, gc) { CASE_EXPECT_EQ(1, wal_obj->gc(now, nullptr)); CASE_EXPECT_EQ(4, wal_obj->get_all_logs().size()); CASE_EXPECT_EQ(old_remove_count + 8, details::g_test_wal_object_stats.event_on_log_removed); -} + auto last_removed_key = (*wal_obj->log_cbegin())->log_key - 1; + CASE_EXPECT_TRUE(wal_obj->get_last_removed_key() && *wal_obj->get_last_removed_key() == last_removed_key); +} CASE_TEST(wal_object, ignore) { test_wal_object_log_storage_type storage; @@ -578,7 +589,6 @@ CASE_TEST(wal_object, ignore) { } while (false); } - CASE_TEST(wal_object, reorder) { test_wal_object_log_storage_type storage; test_wal_object_context ctx; @@ -622,6 +632,6 @@ CASE_TEST(wal_object, reorder) { auto iter = wal_obj->log_begin(); CASE_EXPECT_EQ(log1.get(), (*iter).get()); - ++ iter; + ++iter; CASE_EXPECT_EQ(log2.get(), (*iter).get()); } diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp index 9df11a15..d38eb434 100644 --- a/test/case/wal_publisher_test.cpp +++ b/test/case/wal_publisher_test.cpp @@ -1,10 +1,10 @@ #include +#include #include #include #include #include #include -#include #include @@ -16,6 +16,15 @@ enum class test_wal_publisher_log_action { kFallbackDefault, }; +namespace std { +template <> +struct hash { + std::size_t operator()(test_wal_publisher_log_action const& s) const noexcept { + return std::hash{}(static_cast(s)); + } +}; +} // namespace std + struct test_wal_publisher_log_type { util::distributed_system::wal_time_point timepoint; int64_t log_key; @@ -37,8 +46,8 @@ struct test_wal_publisher_context {}; struct test_wal_publisher_private_type { test_wal_publisher_storage_type* storage; - inline test_wal_publisher_private_type(): storage(nullptr) {} - inline explicit test_wal_publisher_private_type(test_wal_publisher_storage_type* input): storage(input) {} + inline test_wal_publisher_private_type() : storage(nullptr) {} + inline explicit test_wal_publisher_private_type(test_wal_publisher_storage_type* input) : storage(input) {} }; using test_wal_publisher_log_operator = @@ -61,6 +70,8 @@ struct test_wal_publisher_stats { size_t event_on_log_added; size_t event_on_log_removed; + size_t send_logs_count; + size_t send_snapshot_count; size_t send_subscribe_response; size_t event_on_subscribe_added; size_t event_on_subscribe_removed; @@ -73,7 +84,8 @@ struct test_wal_publisher_stats { }; namespace details { -test_wal_publisher_stats g_test_wal_publisher_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, test_wal_publisher_log_type(), nullptr}; +test_wal_publisher_stats g_test_wal_publisher_stats{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, test_wal_publisher_log_type(), + nullptr}; } static test_wal_publisher_type::vtable_pointer create_vtable() { @@ -192,6 +204,7 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { details::g_test_wal_publisher_stats.last_log = *publisher.get_private_data().storage->logs.rbegin(); } + ++ details::g_test_wal_publisher_stats.send_snapshot_count; return wal_result_code::kOk; }; @@ -216,6 +229,7 @@ static test_wal_publisher_type::vtable_pointer create_vtable() { ++details::g_test_wal_publisher_stats.last_event_log_count; } + ++ details::g_test_wal_publisher_stats.send_logs_count; return wal_result_code::kOk; }; @@ -291,3 +305,366 @@ CASE_TEST(wal_publisher, create_failed) { vtable_5->send_logs = nullptr; CASE_EXPECT_EQ(nullptr, test_wal_publisher_type::create(vtable_5, conf, &storage)); } + +CASE_TEST(wal_publisher, load_and_dump) { + auto old_action_count = + details::g_test_wal_publisher_stats.default_action_count + details::g_test_wal_publisher_stats.delegate_action_count; + + test_wal_publisher_storage_type load_storege; + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + load_storege.global_ignore = 123; + load_storege.logs.push_back(test_wal_publisher_log_type{now, 124, test_wal_publisher_log_action::kDoNothing, 124}); + load_storege.logs.push_back(test_wal_publisher_log_type{now, 125, test_wal_publisher_log_action::kFallbackDefault, 125}); + load_storege.logs.push_back(test_wal_publisher_log_type{now, 126, test_wal_publisher_log_action::kRecursivePushBack, 126}); + + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kOk == publisher->load(load_storege, ctx)); + CASE_EXPECT_EQ(old_action_count, details::g_test_wal_publisher_stats.default_action_count + + details::g_test_wal_publisher_stats.delegate_action_count); + CASE_EXPECT_EQ(4, publisher->get_configure().gc_log_size); + CASE_EXPECT_EQ(&publisher->get_log_manager().get_configure(), &publisher->get_configure()); + + CASE_EXPECT_EQ(123, storage.global_ignore); + CASE_EXPECT_EQ(3, storage.logs.size()); + CASE_EXPECT_EQ(124, storage.logs[0].data); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kRecursivePushBack == storage.logs[2].action); + + CASE_EXPECT_EQ(3, publisher->get_log_manager().get_all_logs().size()); + CASE_EXPECT_EQ(124, (*publisher->get_log_manager().get_all_logs().begin())->data); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kRecursivePushBack == (*publisher->get_log_manager().get_all_logs().rbegin())->action); + + // dump + test_wal_publisher_storage_type dump_storege; + CASE_EXPECT_TRUE(util::distributed_system::wal_result_code::kOk == publisher->dump(dump_storege, ctx)); + + CASE_EXPECT_EQ(123, dump_storege.global_ignore); + CASE_EXPECT_EQ(3, dump_storege.logs.size()); + CASE_EXPECT_EQ(124, dump_storege.logs[0].data); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kRecursivePushBack == dump_storege.logs[2].action); + + // check broadcast key + CASE_EXPECT_TRUE(!!publisher->get_broadcast_key_bound()); + if (publisher->get_broadcast_key_bound()) { + CASE_EXPECT_EQ(126, *publisher->get_broadcast_key_bound()); + } +} + +CASE_TEST(wal_publisher, subscriber_basic_operation) { + util::distributed_system::wal_time_point now = std::chrono::system_clock::now(); + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + auto vtable = create_vtable(); + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + + uint64_t subscriber_key_1 = 1; + uint64_t subscriber_key_2 = 2; + uint64_t subscriber_key_3 = 3; + + CASE_EXPECT_EQ(nullptr, publisher->find_subscriber(subscriber_key_1, ctx).get()); + auto subscriber = publisher->create_subscriber(subscriber_key_1, now, ctx, &storage); + CASE_EXPECT_NE(nullptr, subscriber.get()); + CASE_EXPECT_TRUE(now == subscriber->get_last_heartbeat_time_point()); + now += std::chrono::seconds(1); + auto subscriber_copy = publisher->create_subscriber(subscriber_key_1, now, ctx, &storage); + CASE_EXPECT_EQ(subscriber.get(), subscriber_copy.get()); + CASE_EXPECT_TRUE(now == subscriber->get_last_heartbeat_time_point()); + + publisher->create_subscriber(subscriber_key_2, now, ctx, &storage); + publisher->create_subscriber(subscriber_key_3, now, ctx, &storage); + + auto find_result = publisher->find_subscriber(subscriber_key_1, ctx); + CASE_EXPECT_EQ(subscriber.get(), find_result.get()); +} + +static void test_wal_publisher_add_logs(test_wal_publisher_type::object_type& wal_obj, + test_wal_publisher_context ctx, + util::distributed_system::wal_time_point t1, util::distributed_system::wal_time_point t2, + util::distributed_system::wal_time_point t3) { + do { + auto old_default_action_count = details::g_test_wal_publisher_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_publisher_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_publisher_stats.key_alloc; + auto log = wal_obj.allocate_log(t1, test_wal_publisher_log_action::kDoNothing, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kDoNothing == log->action); + CASE_EXPECT_TRUE(t1 == log->timepoint); + + wal_obj.push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_publisher_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 1, details::g_test_wal_publisher_stats.delegate_action_count); + } while (false); + + do { + auto old_default_action_count = details::g_test_wal_publisher_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_publisher_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_publisher_stats.key_alloc; + auto log = wal_obj.allocate_log(t2, test_wal_publisher_log_action::kRecursivePushBack, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kRecursivePushBack == log->action); + CASE_EXPECT_TRUE(t2 == log->timepoint); + + wal_obj.push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count, details::g_test_wal_publisher_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count + 2, details::g_test_wal_publisher_stats.delegate_action_count); + } while (false); + + do { + auto old_default_action_count = details::g_test_wal_publisher_stats.default_action_count; + auto old_delegate_action_count = details::g_test_wal_publisher_stats.delegate_action_count; + + auto previous_key = details::g_test_wal_publisher_stats.key_alloc; + auto log = wal_obj.allocate_log(t3, test_wal_publisher_log_action::kFallbackDefault, ctx); + CASE_EXPECT_TRUE(!!log); + if (!log) { + break; + } + log->data = log->log_key + 100; + CASE_EXPECT_EQ(previous_key + 1, log->log_key); + CASE_EXPECT_TRUE(test_wal_publisher_log_action::kFallbackDefault == log->action); + CASE_EXPECT_TRUE(t3 == log->timepoint); + + wal_obj.push_back(log, ctx); + CASE_EXPECT_EQ(old_default_action_count + 1, details::g_test_wal_publisher_stats.default_action_count); + CASE_EXPECT_EQ(old_delegate_action_count, details::g_test_wal_publisher_stats.delegate_action_count); + } while (false); +} + +CASE_TEST(wal_publisher, subscriber_heartbeat) { + util::distributed_system::wal_time_point t1 = std::chrono::system_clock::now(); + util::distributed_system::wal_time_point t2 = t1 + std::chrono::duration_cast(std::chrono::seconds{3}); + util::distributed_system::wal_time_point t3 = t1 + std::chrono::duration_cast(std::chrono::seconds{6}); + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + if (conf) { + conf->subscriber_timeout = std::chrono::duration_cast(std::chrono::seconds{5}); + } + auto vtable = create_vtable(); + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + + uint64_t subscriber_key_1 = 1; + uint64_t subscriber_key_2 = 2; + uint64_t subscriber_key_3 = 3; + + test_wal_publisher_add_logs(publisher->get_log_manager(), ctx, t1, t2, t3); + + auto event_subscribe_added = details::g_test_wal_publisher_stats.event_on_subscribe_added; + auto event_subscribe_removed = details::g_test_wal_publisher_stats.event_on_subscribe_removed; + auto send_logs_count = details::g_test_wal_publisher_stats.send_logs_count; + auto send_snapshot_count = details::g_test_wal_publisher_stats.send_snapshot_count; + + auto subscriber_1 = publisher->create_subscriber(subscriber_key_1, t1, ctx, &storage); + auto subscriber_2 = publisher->create_subscriber(subscriber_key_2, t2, ctx, &storage); + auto subscriber_3 = publisher->create_subscriber(subscriber_key_3, t3, ctx, &storage); + + CASE_EXPECT_EQ(event_subscribe_added + 3, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + CASE_EXPECT_EQ(send_logs_count, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 3, details::g_test_wal_publisher_stats.send_snapshot_count); + + CASE_EXPECT_GT(publisher->tick(t3, ctx), 0); + + CASE_EXPECT_EQ(event_subscribe_removed + 1, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + + auto all_iter = publisher->get_subscribe_manager().all_range(); + size_t count = 0; + while (all_iter.first != all_iter.second) { + CASE_EXPECT_TRUE(all_iter.first != all_iter.second || count == 2); + ++ all_iter.first; + ++ count; + } + + publisher->remove_subscriber(subscriber_key_1, util::distributed_system::wal_unsubscribe_reason::kClientRequest, ctx); + all_iter = publisher->get_subscribe_manager().all_range(); + count = 0; + while (all_iter.first != all_iter.second) { + CASE_EXPECT_TRUE(all_iter.first != all_iter.second || count == 2); + ++ all_iter.first; + ++ count; + } + CASE_EXPECT_EQ(event_subscribe_removed + 1, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + + publisher->remove_subscriber(subscriber_key_3, util::distributed_system::wal_unsubscribe_reason::kClientRequest, ctx); + all_iter = publisher->get_subscribe_manager().all_range(); + count = 0; + while (all_iter.first != all_iter.second) { + CASE_EXPECT_TRUE(all_iter.first != all_iter.second || count == 2); + ++ all_iter.first; + ++ count; + } + CASE_EXPECT_EQ(event_subscribe_removed + 2, details::g_test_wal_publisher_stats.event_on_subscribe_removed); +} + +CASE_TEST(wal_publisher, subscriber_send_snapshot) { + util::distributed_system::wal_time_point t1 = std::chrono::system_clock::now(); + util::distributed_system::wal_time_point t2 = t1 + std::chrono::duration_cast(std::chrono::seconds{3}); + util::distributed_system::wal_time_point t3 = t1 + std::chrono::duration_cast(std::chrono::seconds{6}); + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + if (conf) { + conf->subscriber_timeout = std::chrono::duration_cast(std::chrono::seconds{5}); + } + auto vtable = create_vtable(); + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + + uint64_t subscriber_key_1 = 1; + + test_wal_publisher_add_logs(publisher->get_log_manager(), ctx, t1, t2, t3); + + auto event_subscribe_added = details::g_test_wal_publisher_stats.event_on_subscribe_added; + auto event_subscribe_removed = details::g_test_wal_publisher_stats.event_on_subscribe_removed; + auto send_logs_count = details::g_test_wal_publisher_stats.send_logs_count; + auto send_snapshot_count = details::g_test_wal_publisher_stats.send_snapshot_count; + + auto subscriber_1 = publisher->create_subscriber(subscriber_key_1, t1, ctx, &storage); + + CASE_EXPECT_EQ(event_subscribe_added + 1, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + CASE_EXPECT_EQ(send_logs_count, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 1, details::g_test_wal_publisher_stats.send_snapshot_count); + + auto last_removed_key = (*publisher->get_log_manager().log_cbegin())->log_key - 1; + publisher->get_log_manager().set_last_removed_key(last_removed_key); + + publisher->receive_subscribe(subscriber_key_1, last_removed_key, t3, ctx); + + CASE_EXPECT_EQ(4, details::g_test_wal_publisher_stats.last_event_log_count); + CASE_EXPECT_EQ(send_logs_count + 1, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 1, details::g_test_wal_publisher_stats.send_snapshot_count); + + publisher->receive_subscribe(subscriber_key_1, last_removed_key - 1, t3, ctx); + CASE_EXPECT_EQ(0, details::g_test_wal_publisher_stats.last_event_log_count); + CASE_EXPECT_EQ(send_logs_count + 1, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 2, details::g_test_wal_publisher_stats.send_snapshot_count); +} + +CASE_TEST(wal_publisher, subscriber_send_logs) { + util::distributed_system::wal_time_point t1 = std::chrono::system_clock::now(); + util::distributed_system::wal_time_point t2 = t1 + std::chrono::duration_cast(std::chrono::seconds{3}); + util::distributed_system::wal_time_point t3 = t1 + std::chrono::duration_cast(std::chrono::seconds{6}); + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + if (conf) { + conf->subscriber_timeout = std::chrono::duration_cast(std::chrono::seconds{5}); + } + auto vtable = create_vtable(); + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + + uint64_t subscriber_key_1 = 1; + + test_wal_publisher_add_logs(publisher->get_log_manager(), ctx, t1, t2, t3); + + auto event_subscribe_added = details::g_test_wal_publisher_stats.event_on_subscribe_added; + auto event_subscribe_removed = details::g_test_wal_publisher_stats.event_on_subscribe_removed; + auto send_logs_count = details::g_test_wal_publisher_stats.send_logs_count; + auto send_snapshot_count = details::g_test_wal_publisher_stats.send_snapshot_count; + + auto subscriber_1 = publisher->create_subscriber(subscriber_key_1, t3, ctx, &storage); + + CASE_EXPECT_EQ(event_subscribe_added + 1, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + CASE_EXPECT_EQ(send_logs_count, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 1, details::g_test_wal_publisher_stats.send_snapshot_count); + + auto from_key = (*publisher->get_log_manager().log_cbegin())->log_key + 1; + + publisher->receive_subscribe(subscriber_key_1, from_key, t3, ctx); + + CASE_EXPECT_EQ(2, details::g_test_wal_publisher_stats.last_event_log_count); + CASE_EXPECT_EQ(send_logs_count + 1, details::g_test_wal_publisher_stats.send_logs_count); + CASE_EXPECT_EQ(send_snapshot_count + 1, details::g_test_wal_publisher_stats.send_snapshot_count); +} + +CASE_TEST(wal_publisher, remove_subscriber_by_check_callback) { + util::distributed_system::wal_time_point t1 = std::chrono::system_clock::now(); + util::distributed_system::wal_time_point t2 = t1 + std::chrono::duration_cast(std::chrono::seconds{3}); + util::distributed_system::wal_time_point t3 = t1 + std::chrono::duration_cast(std::chrono::seconds{6}); + test_wal_publisher_storage_type storage; + test_wal_publisher_context ctx; + + auto conf = create_configure(); + if (conf) { + conf->subscriber_timeout = std::chrono::duration_cast(std::chrono::seconds{5}); + } + + int keep_times = 1; + auto vtable = create_vtable(); + vtable->check_subscriber = [&keep_times](test_wal_publisher_type&, const test_wal_publisher_type::subscriber_pointer&, test_wal_publisher_type::callback_param_type) -> bool { + -- keep_times; + return keep_times >= 0; + }; + + auto publisher = test_wal_publisher_type::create(vtable, conf, &storage); + CASE_EXPECT_TRUE(!!publisher); + if (!publisher) { + return; + } + + uint64_t subscriber_key_1 = 1; + uint64_t subscriber_key_2 = 2; + + test_wal_publisher_add_logs(publisher->get_log_manager(), ctx, t1, t2, t3); + + auto event_subscribe_added = details::g_test_wal_publisher_stats.event_on_subscribe_added; + auto event_subscribe_removed = details::g_test_wal_publisher_stats.event_on_subscribe_removed; + + auto subscriber_1 = publisher->create_subscriber(subscriber_key_1, t2, ctx, &storage); + auto subscriber_2 = publisher->create_subscriber(subscriber_key_2, t3, ctx, &storage); + + CASE_EXPECT_EQ(event_subscribe_added + 2, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + + CASE_EXPECT_NE(nullptr, publisher->find_subscriber(subscriber_key_1, ctx).get()); + CASE_EXPECT_EQ(event_subscribe_added + 2, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + + CASE_EXPECT_EQ(nullptr, publisher->find_subscriber(subscriber_key_1, ctx).get()); + CASE_EXPECT_EQ(event_subscribe_added + 2, details::g_test_wal_publisher_stats.event_on_subscribe_added); + CASE_EXPECT_EQ(event_subscribe_removed + 1, details::g_test_wal_publisher_stats.event_on_subscribe_removed); + + CASE_EXPECT_EQ(subscriber_key_2, publisher->get_subscribe_manager().all_range().first->second->get_key()); +} From 1a11e00dd49361e212da88adcceb69e6e44d750c Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 17 Aug 2021 16:39:37 +0800 Subject: [PATCH 11/12] Fix compile --- include/distributed_system/wal_common_defs.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/include/distributed_system/wal_common_defs.h b/include/distributed_system/wal_common_defs.h index 0b95173c..61b895d8 100644 --- a/include/distributed_system/wal_common_defs.h +++ b/include/distributed_system/wal_common_defs.h @@ -31,19 +31,11 @@ struct LIBATFRAME_UTILS_API_HEAD_ONLY wal_meta_type { ActionCaseT action_case; inline wal_meta_type() = default; -<<<<<<< HEAD template inline wal_meta_type(ToTimepointT&& t, ToLogKeyT&& k, ToActionCaseT&& act) : timepoint(std::forward(t)), log_key(std::forward(k)), action_case(std::forward(act)) {} -======= - template - inline wal_meta_type(ToTimepointT&& t, ToLogKeyT&& k, ToActionCaseT&& act) - : timepoint(std::forward(t)), - log_key(std::forward(k)), - action_case(std::forward(act)) {} ->>>>>>> 213d4f0856b4f57e293736b99b4cafb30f438b72 }; enum class wal_result_code : int32_t { From f80f389bdae445701f37e34d2ff223e329cb8e08 Mon Sep 17 00:00:00 2001 From: owent Date: Tue, 17 Aug 2021 17:09:39 +0800 Subject: [PATCH 12/12] Fix compile on MSVC --- test/case/wal_publisher_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/case/wal_publisher_test.cpp b/test/case/wal_publisher_test.cpp index bf577052..a19f4def 100644 --- a/test/case/wal_publisher_test.cpp +++ b/test/case/wal_publisher_test.cpp @@ -29,7 +29,7 @@ struct test_wal_publisher_log_type { util::distributed_system::wal_time_point timepoint; int64_t log_key; test_wal_publisher_log_action action; - int data; + int64_t data; }; struct test_wal_publisher_storage_type {