增加SST文件的解析模块

Changes to be committed: modified: CMakeLists.txt modified: README.md modified: src/db/db.cpp modified: src/db/db.h modified: src/db/db_impl.cpp modified: src/db/db_impl.h modified: src/db/status.h modified: src/filter/bloom_filter.cpp modified: src/filter/bloom_filter.h modified: src/filter/filter_policy.h new file: src/sst_parser/header.h new file: src/sst_parser/index_block.h new file: src/sst_parser/restart_point.h new file: src/sst_parser/sst_parser.cpp new file: src/sst_parser/sst_parser.h
yangyang233333 · Feb 4, 2023 · 375f991 · 375f991
1 parent ffe43ab
commit 375f991
Show file tree

Hide file tree

Showing 15 changed files with 439 additions and 14 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -44,6 +44,8 @@ file(GLOB_RECURSE SRC
         src/memtable/*.h
         src/recovery/*.cpp
         src/recovery/*.h
+        src/sst_parser/*.cpp
+        src/sst_parser/*.h
         src/table/*.cpp
         src/table/*.h
         src/utils/*.cpp
@@ -73,6 +75,8 @@ file(GLOB_RECURSE SRC_WITHOUT_MAIN
         src/memtable/*.h
         src/recovery/*.cpp
         src/recovery/*.h
+        src/sst_parser/*.cpp
+        src/sst_parser/*.h
         src/table/*.cpp
         src/table/*.h
         src/utils/*.cpp

diff --git a/README.md b/README.md
@@ -12,6 +12,7 @@ smallkv 是一个列存的、基于LSM架构的存储引擎。
 
 **项目正在疯狂迭代中！！**
 
+**Attention**: This is a toy project, so it cannot be used in production environment.
 ---
 
 ## 客户端演示
@@ -137,6 +138,5 @@ MetaBlock_OffsetInfo记录了MetaBlock的size和offset，IndexBlock_OffsetInfo
 7. [Linux I/O: fsync, fflush, fwrite, mmap](https://juejin.cn/post/7001665675907301412)
 
 ---
-
-感谢 [JetBrains](https://jb.gg/OpenSourceSupport) 捐献的免费许可证帮助我们开发smallkv。  
+
 Thanks to [JetBrains](https://jb.gg/OpenSourceSupport) for donating product licenses to help develop **smallkv** <a href="https://jb.gg/OpenSourceSupport"><img src="img/jb_beam.svg" width="94" align="center" /></a>
diff --git a/src/db/db.cpp b/src/db/db.cpp
@@ -22,8 +22,8 @@ namespace smallkv {
 
     DBStatus DB::Get(const ReadOptions &options,
                      const std::string_view &key,
-                     std::string *value) {
-        return db_impl->Get(options, key, value);
+                     std::string *ret_value_ptr) {
+        return db_impl->Get(options, key, ret_value_ptr);
     }
 
     DBStatus DB::BatchPut(const WriteOptions &options) {

diff --git a/src/db/db.h b/src/db/db.h
@@ -32,7 +32,7 @@ namespace smallkv {
         // 将Key对应的值写到value地址上
         DBStatus Get(const ReadOptions &options,
                      const std::string_view &key,
-                     std::string *value);
+                     std::string *ret_value_ptr);
 
         // 批写
         DBStatus BatchPut(const WriteOptions &options);

diff --git a/src/db/db_impl.cpp b/src/db/db_impl.cpp
@@ -12,6 +12,7 @@
 #include "wal/wal_writer.h"
 #include "file/file_writer.h"
 #include "table/sstable_builder.h"
+#include "sst_parser/sst_parser.h"
 
 namespace smallkv {
     DBImpl::DBImpl(Options options) : options_(std::move(options)) {
@@ -110,7 +111,7 @@ namespace smallkv {
 
     DBStatus DBImpl::Get(const ReadOptions &options,
                          const std::string_view &key,
-                         std::string *value) {
+                         std::string *ret_value_ptr) {
         assert(closed == false);
         /*
          * 读逻辑：
@@ -124,25 +125,35 @@ namespace smallkv {
 
         // 1. 读缓存
         if (cache->contains(key.data())) {
-            *value = *(cache->get(key.data())->val);
+            *ret_value_ptr = *(cache->get(key.data())->val);
+            logger->info("Cache hit.");
             return Status::Success;
         }
 
         // 2. 读memtable
         if (mem_table->Contains(key)) {
             auto val = mem_table->Get(key);
-            *value = mem_table->Get(key.data()).value();
+            *ret_value_ptr = mem_table->Get(key.data()).value();
             return Status::Success;
         }
 
         // 3. 依次读sst文件
-        // todo: 后续实现
+        //todo: 实际上需要从manifest获取sst文件，此处直接硬编码一个文件: level_0_sst_0.sst
+        std::string sst_path = options_.STORAGE_DIR + "level_0_sst_0.sst";
+        auto sst_parser = SSTParser(sst_path);
+        sst_parser.Parse();
+        auto val = sst_parser.Seek(key);
+        if (!val.has_value()) {
+            return Status::NotFound;
+        }
+        ret_value_ptr->append(val.value());
 
         // 4. 找到的数据写入缓存
-        // todo
-
+        //todo: 这里cache保存的是value的指针，然而val可能是临时值，需要优化，此处临时new 一下，性能拉胯
+        auto val_ = new std::string(val.value());
+        cache->insert(key.data(), val_);
 
-        return Status::ExecFailed;
+        return Status::Success;
     }
 
     DBStatus DBImpl::BatchPut(const WriteOptions &options) {

diff --git a/src/db/db_impl.h b/src/db/db_impl.h
@@ -42,7 +42,7 @@ namespace smallkv {
         // 将Key对应的值写到value地址上
         DBStatus Get(const ReadOptions &options,
                      const std::string_view &key,
-                     std::string *value);
+                     std::string *ret_value_ptr);
 
         // 关闭数据库：调用此函数可以保证所有已写入数据会被持久化到磁盘，
         DBStatus Close();

diff --git a/src/db/status.h b/src/db/status.h
@@ -25,6 +25,7 @@ namespace smallkv {
         static constexpr DBStatus InvalidArgs = {2, "Invalid args."};
         static constexpr DBStatus ExecFailed = {3, "Exec failed."};
         static constexpr DBStatus NotImpl = {4, "Not implemented."};
+        static constexpr DBStatus NotFound = {5, "Key not found."};
     };
 }
 

diff --git a/src/filter/bloom_filter.cpp b/src/filter/bloom_filter.cpp
@@ -68,4 +68,9 @@ namespace smallkv {
     std::string &BloomFilter::data() {
         return bits_array;
     }
+
+    void BloomFilter::create_filter2(int32_t hash_func_num_, std::string &bits_array_) {
+        hash_func_num = hash_func_num_;
+        bits_array = bits_array_;
+    }
 };
diff --git a/src/filter/bloom_filter.h b/src/filter/bloom_filter.h
@@ -12,7 +12,8 @@
 namespace smallkv {
     class BloomFilter final : public FilterPolicy {
     public:
-        BloomFilter() = delete;
+        // 主要用于读取sst的时候创建filter
+        BloomFilter() = default;
 
 //        bits_per_key指每个key所占据的位数
 //        explicit BloomFilter(int32_t bits_per_key);
@@ -39,6 +40,9 @@ namespace smallkv {
         // keys: 待插入的数据
         void create_filter(const std::vector<std::string> &keys) override;
 
+        // 主要用于读取sst的时候创建filter
+        void create_filter2(int32_t hash_func_num_, std::string &bits_array_) override;
+
         // 判断元素key是否存在
         // 返回值： false=不存在；true=可能存在；
         bool exists(const std::string_view &key) override;

diff --git a/src/filter/filter_policy.h b/src/filter/filter_policy.h
@@ -27,6 +27,10 @@ namespace smallkv {
 
         virtual void create_filter(const std::vector<std::string> &keys) = 0;
 
+        // 主要用于读取sst的时候创建filter
+        virtual void create_filter2(int32_t hash_func_num_,
+                                    std::string &bits_array_) = 0;
+
         // 检查是否存在，如果为false表示一定不存在，true表示可能存在
         virtual bool exists(const std::string_view &key) = 0;
 

diff --git a/src/sst_parser/header.h b/src/sst_parser/header.h
@@ -0,0 +1,15 @@
+//
+// Created by qianyy on 2023/1/30.
+//
+#include "db/offset_info.h"
+
+#ifndef SMALLKV_HEADER_H
+#define SMALLKV_HEADER_H
+namespace smallkv {
+    // SST中Header的定义
+    struct Header {
+        OffsetInfo MetaBlock_OffsetInfo;
+        OffsetInfo IndexBlock_OffsetInfo;
+    };
+}
+#endif //SMALLKV_HEADER_H
diff --git a/src/sst_parser/index_block.h b/src/sst_parser/index_block.h
@@ -0,0 +1,16 @@
+//
+// Created by qianyy on 2023/1/31.
+//
+#include <stdint.h>
+#include <string>
+#include "db/offset_info.h"
+
+#ifndef SMALLKV_INDEX_BLOCK_H
+#define SMALLKV_INDEX_BLOCK_H
+namespace smallkv {
+    struct IndexBlock {
+        std::string _shortest_key; // 保证 _shortest_key >= 对应的DataBlock中的最大key
+        OffsetInfo offsetInfo;     // 对应DataBlock的offset信息
+    };
+}
+#endif //SMALLKV_INDEX_BLOCK_H
diff --git a/src/sst_parser/restart_point.h b/src/sst_parser/restart_point.h
@@ -0,0 +1,16 @@
+//
+// Created by qianyy on 2023/2/4.
+//
+#include "db/offset_info.h"
+#include <string>
+
+#ifndef SMALLKV_RESTART_POINT_H
+#define SMALLKV_RESTART_POINT_H
+namespace smallkv {
+    struct RestartPoint {
+        int record_num = 0;
+        OffsetInfo rp_offset;
+        std::string fullkey;
+    };
+}
+#endif //SMALLKV_RESTART_POINT_H