diff --git a/README.md b/README.md index 068460b..c8ec55f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ smallkv 是一个列存的、基于LSM架构的存储引擎。 - [x] SSTable - [ ] MANIFEST - [x] WAL模块 -- [ ] memtable +- [x] memtable - [ ] 读流程 - [ ] 写流程 - [ ] Compaction模块 diff --git a/src/memtable/memtable.cpp b/src/memtable/memtable.cpp new file mode 100644 index 0000000..362bdb2 --- /dev/null +++ b/src/memtable/memtable.cpp @@ -0,0 +1,46 @@ +// +// Created by qianyy on 2023/1/25. +// +#include "skiplist.h" +#include "memtable.h" +#include "utils/codec.h" +#include "log/log.h" + +namespace smallkv { + MemTable::MemTable(std::shared_ptr alloc) : alloc(std::move(alloc)) { + // todo: 优化点:此处应该使用string_view + ordered_table_ = std::make_shared>(alloc); + logger = log::get_logger(); + } + + void MemTable::Insert(OpType op_type, + const std::string_view &key, + const std::string_view &value) { + if (op_type == OpType::kAdd) { + // 新数据插入 + ordered_table_->Insert(key.data(), value.data()); + } else if (op_type == OpType::kUpdate) { + // todo: 对于原地更新,最好在skiplist中直接提供接口,而不是采用Del-Add的低效方法。 + // 旧数据原地更新 + ordered_table_->Delete(key.data()); + ordered_table_->Insert(key.data(), value.data()); + } else if (op_type == OpType::kDeletion) { + // todo 同理,建议直接在skiplist中提供接口 + if (ordered_table_->Contains(key.data())) { + ordered_table_->Delete(key.data()); + } + ordered_table_->Insert(key.data(), ""); // value="" 表示删除 + } else { + logger->error("Unexpected op_type. op_type={}", op_type); + return; + } + } + + bool MemTable::Contains(const std::string_view &key) { + return ordered_table_->Contains(key.data()); + } + + std::optional MemTable::Get(const std::string_view &key) { + return ordered_table_->Get(key.data()); + } +} \ No newline at end of file diff --git a/src/memtable/memtable.h b/src/memtable/memtable.h new file mode 100644 index 0000000..32a492f --- /dev/null +++ b/src/memtable/memtable.h @@ -0,0 +1,77 @@ +// +// Created by qianyy on 2023/1/25. +// +#include +#include +#include "memtable_config.h" +#include "op_type.h" +#include "log/log.h" + +#ifndef SMALLKV_MEMTABLE_H +#define SMALLKV_MEMTABLE_H + +namespace smallkv { + template + class SkipList; + + class FreeListAllocate; + + /* + * Insert逻辑: + * 1. Add key, OpType=kAdd + * 1.1 active memtable中不存在key,正常添加即可; + * 1.2 active memtable中存在key,进行原地更新。 + * + * 2. Delete key, OpType=kDeletion + * 2.1 active memtable中不存在key,相当于插入 + * 2.2 active memtable中存在key,原地更新为 + * + * + * */ + class MemTable final { + public: + explicit MemTable(std::shared_ptr alloc); + + ~MemTable() = default; + + MemTable() = delete; + + // 禁用拷贝、赋值。 + MemTable(const MemTable &) = delete; + + MemTable &operator=(const MemTable &) = delete; + + inline void Add(const std::string_view &key, + const std::string_view &value) { + this->Insert(OpType::kAdd, key, value); + } + + inline void Update(const std::string_view &key, + const std::string_view &value) { + this->Insert(OpType::kUpdate, key, value); + } + + inline void Delete(const std::string_view &key) { + this->Insert(OpType::kDeletion, key, ""); + } + + bool Contains(const std::string_view &key); + + // 如果不存在则返回nullopt + std::optional Get(const std::string_view &key); + + private: + // Add、Update、Delete都属于Insert + // 如果是Delete,则value="" + // OpType = {kDeletion, kAdd, kUpdate} + void Insert(OpType op_type, const std::string_view &key, + const std::string_view &value); + + private: + std::shared_ptr> ordered_table_; + std::shared_ptr alloc; + std::shared_ptr logger; + }; +} + +#endif //SMALLKV_MEMTABLE_H diff --git a/src/memtable/memtable_config.h b/src/memtable/memtable_config.h new file mode 100644 index 0000000..fedc640 --- /dev/null +++ b/src/memtable/memtable_config.h @@ -0,0 +1,14 @@ +// +// Created by qianyy on 2023/1/25. +// + +#ifndef SMALLKV_MEMTABLE_CONFIG_H +#define SMALLKV_MEMTABLE_CONFIG_H +namespace smallkv { + struct MemTableConfig { + MemTableConfig() = delete; + + static constexpr int MAX_KEY_NUM = 4096; // 超过4096个Key,就新建一个memtable + }; +} +#endif //SMALLKV_MEMTABLE_CONFIG_H diff --git a/src/memtable/op_type.h b/src/memtable/op_type.h new file mode 100644 index 0000000..bb8f82e --- /dev/null +++ b/src/memtable/op_type.h @@ -0,0 +1,14 @@ +// +// Created by qianyy on 2023/1/26. +// + +#ifndef SMALLKV_OP_TYPE_H +#define SMALLKV_OP_TYPE_H +namespace smallkv { + enum OpType { + kDeletion = 0x1, // 删除 + kAdd = 0x2, // 添加 + kUpdate = 0x3, // 更新 + }; +} +#endif //SMALLKV_OP_TYPE_H diff --git a/src/memtable/skiplist.h b/src/memtable/skiplist.h index 732fd6b..c1911f8 100644 --- a/src/memtable/skiplist.h +++ b/src/memtable/skiplist.h @@ -2,10 +2,11 @@ // Created by qianyy on 2023/1/23. // #include +#include #include #include -#include #include +#include #include "log/log.h" #include "memory/allocate.h" #include "skiplist_config.h" @@ -20,15 +21,15 @@ namespace smallkv { * 注:线程不安全、不支持重复的key插入 * * */ - template + template class SkipList { class Node; public: explicit SkipList(std::shared_ptr alloc); - // 插入key - void Insert(const Key &key); + // 插入一个新的key + void Insert(const Key &key, const Value &value); // 删除key void Delete(const Key &key); @@ -36,6 +37,9 @@ namespace smallkv { // 存在key则返回true bool Contains(const Key &key); + // 注:如果要找的key不存在,则返回nullopt + std::optional Get(const Key &key); + // 仅用于DEBUG:打印表 void OnlyUsedForDebugging_Print_() { auto p = head_->next[0]; @@ -47,6 +51,8 @@ namespace smallkv { std::cout << "============= DEBUG =============" << std::endl; } + inline int GetSize() { return size; } + private: int RandomLevel(); @@ -55,7 +61,7 @@ namespace smallkv { // 找到key节点的前缀节点,也就是找到key的待插入位置 void FindPrevNode(const Key &key, std::vector &prev); - inline Node *NewNode(const Key &key, int level); + inline Node *NewNode(const Key &key, int level, const Value &value); private: Node *head_; // 头结点,高度为SkipListConfig::kMaxHeight,不存数据 @@ -68,8 +74,38 @@ namespace smallkv { std::shared_ptr logger = log::get_logger(); }; - template - void SkipList::Delete(const Key &key) { + template + std::optional SkipList::Get(const Key &key) { + int level = GetCurrentHeight() - 1; + auto cur = head_; + while (true) { + auto next = cur->next[level]; + if (next == nullptr) { + if (level == 0) { + // 遍历到这里说明key不存在 + return std::nullopt; + } else { + --level; + } + } else { + if (next->key == key) { + return next->value; // 找到了 + } else if (next->key < key) { + cur = next; + } else if (next->key > key) { + if (level == 0) { + // 遍历到这里说明key不存在 + return std::nullopt; + } else { + --level; // 在非最底层遇到了大于key的数,应该下降 + } + } + } + } + } + + template + void SkipList::Delete(const Key &key) { if (Contains(key) == false) { logger->warn("The value you want to delete does not exist. Key={}", key); return; @@ -114,7 +150,7 @@ namespace smallkv { } // assert(level_of_target_node > 0); // assert(level_of_target_node <= prev.size()); - logger->info("level_of_target_node={}", level_of_target_node); +// logger->info("level_of_target_node={}", level_of_target_node); for (int i = 0; i < level_of_target_node; ++i) { if (prev[i] != nullptr) { assert(prev[i]->next[i] != nullptr); @@ -123,8 +159,8 @@ namespace smallkv { } } - template - bool SkipList::Contains(const Key &key) { // 存在key则返回true + template + bool SkipList::Contains(const Key &key) { // 存在key则返回true int level = GetCurrentHeight() - 1; auto cur = head_; while (true) { @@ -151,8 +187,8 @@ namespace smallkv { } } - template - void SkipList::Insert(const Key &key) { + template + void SkipList::Insert(const Key &key, const Value &value) { if (Contains(key)) { logger->warn("A duplicate key was inserted. Key={}", key); return; @@ -168,7 +204,7 @@ namespace smallkv { FindPrevNode(key, prev); int level_of_new_node = RandomLevel(); max_level = std::max(level_of_new_node, max_level); // 更新最大高度 - auto newNode = NewNode(key, level_of_new_node); + auto newNode = NewNode(key, level_of_new_node, value); for (int i = 0; i < newNode->GetLevel(); ++i) { if (prev[i] == nullptr) { @@ -181,20 +217,20 @@ namespace smallkv { } } - template - int SkipList::GetCurrentHeight() { + template + int SkipList::GetCurrentHeight() { return max_level; } - template - typename SkipList::Node *SkipList::NewNode(const Key &key, int level) { + template + typename SkipList::Node *SkipList::NewNode(const Key &key, int level, const Value &value) { // todo: 不确定FreeListAllocate实现有没有问题, // 所以此处先使用系统allocator,稳定了再换。 - return new Node(key, level); + return new Node(key, level, value); } - template - void SkipList::FindPrevNode( + template + void SkipList::FindPrevNode( const Key &key, std::vector &prev) { int level = GetCurrentHeight() - 1; auto cur = head_; @@ -213,8 +249,8 @@ namespace smallkv { } } - template - int SkipList::RandomLevel() { + template + int SkipList::RandomLevel() { int level = 1; while (level < SkipListConfig::kMaxHeight && rand() & 1) { ++level; @@ -222,19 +258,21 @@ namespace smallkv { return level; } - template - SkipList::SkipList(std::shared_ptr alloc) + template + SkipList::SkipList(std::shared_ptr alloc) :alloc(std::move(alloc)) { srand(time(0)); - head_ = NewNode("", SkipListConfig::kMaxHeight); + head_ = NewNode("", SkipListConfig::kMaxHeight, ""); max_level = 1; size = 0; } - template - class SkipList::Node { + template + class SkipList::Node { public: - Node(const Key &key, int level) : key(key) { + Node() = delete; + + Node(const Key &key, int level, const Value &value) : key(key), value(value) { next.resize(level, nullptr); } @@ -243,6 +281,7 @@ namespace smallkv { inline int GetLevel() { return next.size(); } const Key key; + Value value; std::vector next; }; } diff --git a/src/utils/codec.cpp b/src/utils/codec.cpp index 794b202..6cce4f3 100644 --- a/src/utils/codec.cpp +++ b/src/utils/codec.cpp @@ -5,6 +5,9 @@ #include "codec.h" namespace smallkv::utils { + void EncodeFixed8(char *buf, uint8_t val) { + buf[0] = val & 0xff; + } void EncodeFixed32(char *buf, uint32_t val) { buf[0] = val & 0xff; @@ -24,6 +27,10 @@ namespace smallkv::utils { buf[7] = (val >> 56) & 0xff; } + uint8_t DecodeFixed8(const char *data) { + return *reinterpret_cast(data); + } + uint32_t DecodeFixed32(const char *data) { auto _data = reinterpret_cast(data); return static_cast(_data[0]) | diff --git a/src/utils/codec.h b/src/utils/codec.h index 5fe7bee..684decc 100644 --- a/src/utils/codec.h +++ b/src/utils/codec.h @@ -8,14 +8,24 @@ #define SMALLKV_CODEC_H namespace smallkv::utils { // 编解码 + void EncodeFixed8(char *buf, uint8_t val); + void EncodeFixed32(char *buf, uint32_t val); void EncodeFixed64(char *buf, uint64_t val); + uint8_t DecodeFixed8(const char *data); + uint32_t DecodeFixed32(const char *data); uint64_t DecodeFixed64(const char *data); + inline void PutFixed8(std::string &dst, uint8_t val) { + char buf[sizeof(val)]; + EncodeFixed8(buf, val); + dst.append(buf, sizeof(val)); + } + inline void PutFixed32(std::string &dst, uint32_t val) { char buf[sizeof(val)]; EncodeFixed32(buf, val); diff --git a/tests/test_memtable.cpp b/tests/test_memtable.cpp new file mode 100644 index 0000000..76ec2ff --- /dev/null +++ b/tests/test_memtable.cpp @@ -0,0 +1,80 @@ +// +// Created by qianyy on 2023/1/26. +// +#include +#include +#include +#include +#include +#include "memtable/memtable.h" +#include "memory/allocate.h" + +namespace smallkv { + TEST(MemTable, Insert__Add) { + auto alloc = std::make_shared(); + auto mem_table = std::make_shared(alloc); + + constexpr int N = 1234; + for (int i = 0; i < N; ++i) { + auto key = "key_" + std::to_string(i); + auto val = "val_" + std::to_string(i); + mem_table->Add(key, val); + } + for (int i = 0; i < N; ++i) { + auto key = "key_" + std::to_string(i); + auto val = "val_" + std::to_string(i); + EXPECT_EQ(mem_table->Get(key), val); + } + } + + // 插入重复值 + TEST(MemTable, Insert__add_update) { + auto alloc = std::make_shared(); + auto mem_table = std::make_shared(alloc); + + srand(time(nullptr)); + constexpr int N = 1234; + + for (int i = 0; i < N; ++i) { + auto key = "key_" + std::to_string(i); + auto val = "val_" + std::to_string(i); + mem_table->Add(key, val); + // 随机插入重复值 + if (rand() & 1) { + auto duplicate_val = "duplicate_" + val; + mem_table->Update(key, duplicate_val); + EXPECT_EQ(mem_table->Get(key), duplicate_val); + } else { + EXPECT_EQ(mem_table->Get(key), val); + } + } + } + + TEST(MemTable, Insert__Deletion) { + auto alloc = std::make_shared(); + auto mem_table = std::make_shared(alloc); + + srand(time(nullptr)); + constexpr int N = 1234; + + for (int i = 0; i < N; ++i) { + auto key = "key_" + std::to_string(i); + auto val = "val_" + std::to_string(i); + mem_table->Add(key, val); + // 随机插入重复值 + if (rand() & 1) { + auto duplicate_val = "duplicate_" + val; + mem_table->Update(key, duplicate_val); + EXPECT_EQ(mem_table->Get(key), duplicate_val); + } else { + EXPECT_EQ(mem_table->Get(key), val); + } + + // 随机删除 + if (rand() & 1) { + mem_table->Delete(key); + EXPECT_EQ(mem_table->Get(key), ""); + } + } + } +} diff --git a/tests/test_skiplist.cpp b/tests/test_skiplist.cpp index 2d093d3..2bd4ecc 100644 --- a/tests/test_skiplist.cpp +++ b/tests/test_skiplist.cpp @@ -12,29 +12,29 @@ namespace smallkv { TEST(skiplist, Insert) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); - skiplist->Insert("1"); - skiplist->Insert("2"); + std::shared_ptr> skiplist = + std::make_shared>(alloc); + skiplist->Insert("1", "value_1"); + skiplist->Insert("2", "value_2"); } TEST(skiplist, Insert2) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); + std::shared_ptr> skiplist = + std::make_shared>(alloc); for (int i = 0; i < 100; ++i) { - skiplist->Insert(std::to_string(i)); + skiplist->Insert(std::to_string(i), "value_" + std::to_string(i)); } } TEST(skiplist, Contains) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); + std::shared_ptr> skiplist = + std::make_shared>(alloc); - skiplist->Insert("1"); - skiplist->Insert("3"); - skiplist->Insert("5"); + skiplist->Insert("1", "value_1"); + skiplist->Insert("3", "value_3"); + skiplist->Insert("5", "value_5"); EXPECT_EQ(skiplist->Contains("1"), true); EXPECT_EQ(skiplist->Contains("3"), true); EXPECT_EQ(skiplist->Contains("5"), true); @@ -46,8 +46,8 @@ namespace smallkv { TEST(skiplist, Contains2) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); + std::shared_ptr> skiplist = + std::make_shared>(alloc); const int N = 2000; srand(time(0)); @@ -61,7 +61,7 @@ namespace smallkv { } } for (const auto &ye: yes) { - skiplist->Insert(ye); + skiplist->Insert(ye, "value_" + ye); } for (const auto &y: yes) { @@ -74,12 +74,12 @@ namespace smallkv { TEST(skiplist, Delete) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); + std::shared_ptr> skiplist = + std::make_shared>(alloc); - skiplist->Insert("1"); - skiplist->Insert("3"); - skiplist->Insert("5"); + skiplist->Insert("1", "value_1"); + skiplist->Insert("3", "value_3"); + skiplist->Insert("5", "value_5"); EXPECT_EQ(skiplist->Contains("1"), true); EXPECT_EQ(skiplist->Contains("3"), true); EXPECT_EQ(skiplist->Contains("5"), true); @@ -94,11 +94,11 @@ namespace smallkv { TEST(skiplist, Delete2) { auto alloc = std::make_shared(); - std::shared_ptr> skiplist = - std::make_shared>(alloc); + std::shared_ptr> skiplist = + std::make_shared>(alloc); const int N = 2000; for (int i = 0; i < N; ++i) { - skiplist->Insert(std::to_string(i)); + skiplist->Insert(std::to_string(i), "value_" + std::to_string(i)); int flag = rand() & 0x1; if (flag == 1) { skiplist->Delete(std::to_string(i)); @@ -108,4 +108,41 @@ namespace smallkv { } } } + + TEST(skiplist, Get) { + auto alloc = std::make_shared(); + std::shared_ptr> skiplist = + std::make_shared>(alloc); + + skiplist->Insert("1", "value_1"); + skiplist->Insert("3", "value_3"); + skiplist->Insert("5", "value_5"); + + EXPECT_EQ(skiplist->Get("0"), std::nullopt); + EXPECT_EQ(skiplist->Get("1"), "value_1"); + EXPECT_EQ(skiplist->Get("2"), std::nullopt); + EXPECT_EQ(skiplist->Get("3"), "value_3"); + EXPECT_EQ(skiplist->Get("4"), std::nullopt); + EXPECT_EQ(skiplist->Get("5"), "value_5"); + } + + TEST(skiplist, Get2) { + auto alloc = std::make_shared(); + std::shared_ptr> skiplist = + std::make_shared>(alloc); + const int N = 1234; + for (int i = 0; i < N; ++i) { + skiplist->Insert(std::to_string(i), "value_" + std::to_string(i)); + if (i & 1) { + skiplist->Delete(std::to_string(i)); + } + } + for (int i = 0; i < N; ++i) { + if (i & 1) { + EXPECT_EQ(skiplist->Get(std::to_string(i)), std::nullopt); + } else { + EXPECT_EQ(skiplist->Get(std::to_string(i)), "value_" + std::to_string(i)); + } + } + } }