diff --git a/be/src/http/action/file_cache_action.cpp b/be/src/http/action/file_cache_action.cpp index 740bac46edf2a7f..c973db7edb3001c 100644 --- a/be/src/http/action/file_cache_action.cpp +++ b/be/src/http/action/file_cache_action.cpp @@ -50,6 +50,8 @@ constexpr static std::string_view CLEAR = "clear"; constexpr static std::string_view RESET = "reset"; constexpr static std::string_view HASH = "hash"; constexpr static std::string_view LIST_CACHE = "list_cache"; +constexpr static std::string_view LIST_BASE_PATHS = "list_base_paths"; +constexpr static std::string_view CHECK_CONSISTENCY = "check_consistency"; constexpr static std::string_view CAPACITY = "capacity"; constexpr static std::string_view RELEASE = "release"; constexpr static std::string_view BASE_PATH = "base_path"; @@ -127,6 +129,25 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri *json_metrics = json.ToString(); } } + } else if (operation == LIST_BASE_PATHS) { + auto all_cache_base_path = io::FileCacheFactory::instance()->get_base_paths(); + EasyJson json; + std::for_each(all_cache_base_path.begin(), all_cache_base_path.end(), + [&json](auto& x) { json.PushBack(std::move(x)); }); + *json_metrics = json.ToString(); + } else if (operation == CHECK_CONSISTENCY) { + const std::string& cache_base_path = req->param(BASE_PATH.data()); + if (cache_base_path.empty()) { + st = Status::InvalidArgument("missing parameter: {} is required", VALUE.data()); + } else { + auto* block_file_cache = io::FileCacheFactory::instance()->get_by_path(cache_base_path); + std::vector inconsistencies; + RETURN_IF_ERROR(block_file_cache->report_file_cache_inconsistency(inconsistencies)); + EasyJson json; + std::for_each(inconsistencies.begin(), inconsistencies.end(), + [&json](auto& x) { json.PushBack(std::move(x)); }); + *json_metrics = json.ToString(); + } } else { st = Status::InternalError("invalid operation: {}", operation); } @@ -145,4 +166,4 @@ void FileCacheAction::handle(HttpRequest* req) { } } -} // namespace doris +} // namespace doris \ No newline at end of file diff --git a/be/src/io/cache/block_file_cache.cpp b/be/src/io/cache/block_file_cache.cpp index ebcbe9135daa351..275d715b7218bce 100644 --- a/be/src/io/cache/block_file_cache.cpp +++ b/be/src/io/cache/block_file_cache.cpp @@ -20,6 +20,8 @@ #include "io/cache/block_file_cache.h" +#include + #include "common/status.h" #include "cpp/sync_point.h" @@ -2038,4 +2040,84 @@ std::map BlockFileCache::get_stats_unsafe() { template void BlockFileCache::remove(FileBlockSPtr file_block, std::lock_guard& cache_lock, std::lock_guard& block_lock, bool sync); + +Status BlockFileCache::report_file_cache_inconsistency(std::vector& results) { + InconsistencyContext inconsistency_context; + RETURN_IF_ERROR(check_file_cache_consistency(inconsistency_context)); + auto n = inconsistency_context.types.size(); + results.reserve(n); + for (size_t i = 0; i < n; i++) { + std::string result; + result += "File cahce info in manager:\n"; + result += inconsistency_context.infos_in_manager[i].to_string(); + result += "File cahce info in storage:\n"; + result += inconsistency_context.infos_in_storage[i].to_string(); + result += inconsistency_context.types[i].to_string(); + result += "\n"; + results.push_back(std::move(result)); + } + return Status::OK(); +} + +Status BlockFileCache::check_file_cache_consistency(InconsistencyContext& inconsistency_context) { + std::lock_guard cache_lock(_mutex); + std::vector infos_in_storage; + RETURN_IF_ERROR(_storage->get_file_cache_infos(infos_in_storage, cache_lock)); + std::unordered_set confirmed_blocks; + for (const auto& info_in_storage : infos_in_storage) { + confirmed_blocks.insert({info_in_storage.hash, info_in_storage.offset}); + auto* cell = get_cell(info_in_storage.hash, info_in_storage.offset, cache_lock); + if (cell == nullptr) { + inconsistency_context.infos_in_manager.emplace_back(); + inconsistency_context.infos_in_storage.push_back(info_in_storage); + inconsistency_context.types.emplace_back(InconsistencyType::NOT_LOADED); + continue; + } + FileCacheInfo info_in_manager { + .hash = info_in_storage.hash, + .expiration_time = cell->file_block->expiration_time(), + .size = cell->size(), + .offset = info_in_storage.offset, + .is_tmp = cell->file_block->state() == FileBlock::State::DOWNLOADING, + .cache_type = cell->file_block->cache_type()}; + InconsistencyType inconsistent_type; + if (info_in_storage.is_tmp != info_in_manager.is_tmp) { + inconsistent_type |= InconsistencyType::TMP_FILE_EXPECT_DOWNLOADING_STATE; + } + size_t expected_size = + info_in_manager.is_tmp ? cell->dowloading_size() : info_in_manager.size; + if (info_in_storage.size != expected_size) { + inconsistent_type |= InconsistencyType::SIZE_INCONSISTENT; + } + // Only if it is not a tmp file need we check the cache type. + if ((inconsistent_type & InconsistencyType::TMP_FILE_EXPECT_DOWNLOADING_STATE) == 0 && + info_in_storage.cache_type != info_in_manager.cache_type) { + inconsistent_type |= InconsistencyType::CACHE_TYPE_INCONSISTENT; + } + if (info_in_storage.expiration_time != info_in_manager.expiration_time) { + inconsistent_type |= InconsistencyType::EXPIRATION_TIME_INCONSISTENT; + } + if (inconsistent_type != InconsistencyType::NONE) { + inconsistency_context.infos_in_manager.push_back(info_in_manager); + inconsistency_context.infos_in_storage.push_back(info_in_storage); + inconsistency_context.types.push_back(inconsistent_type); + } + } + + for (const auto& [hash, offset_to_cell] : _files) { + for (const auto& [offset, cell] : offset_to_cell) { + if (confirmed_blocks.contains({hash, offset})) { + continue; + } + const auto& block = cell.file_block; + inconsistency_context.infos_in_manager.emplace_back( + hash, block->expiration_time(), cell.size(), offset, + cell.file_block->state() == FileBlock::State::DOWNLOADING, block->cache_type()); + inconsistency_context.infos_in_storage.emplace_back(); + inconsistency_context.types.emplace_back(InconsistencyType::MISSING_IN_STORAGE); + } + } + return Status::OK(); +} + } // namespace doris::io diff --git a/be/src/io/cache/block_file_cache.h b/be/src/io/cache/block_file_cache.h index f23d5a3799e0cfe..22f7f9924b542dd 100644 --- a/be/src/io/cache/block_file_cache.h +++ b/be/src/io/cache/block_file_cache.h @@ -329,6 +329,9 @@ class BlockFileCache { using QueryFileCacheContextHolderPtr = std::unique_ptr; QueryFileCacheContextHolderPtr get_query_context_holder(const TUniqueId& query_id); + Status report_file_cache_inconsistency(std::vector& results); + Status check_file_cache_consistency(InconsistencyContext& inconsistency_context); + private: struct FileBlockCell { FileBlockSPtr file_block; @@ -349,6 +352,7 @@ class BlockFileCache { bool releasable() const { return file_block.use_count() == 1; } size_t size() const { return file_block->_block_range.size(); } + size_t dowloading_size() const { return file_block->_downloaded_size; } FileBlockCell(FileBlockSPtr file_block, std::lock_guard& cache_lock); FileBlockCell(FileBlockCell&& other) noexcept diff --git a/be/src/io/cache/file_cache_common.cpp b/be/src/io/cache/file_cache_common.cpp index 19041938a08346d..5045e406d82794a 100644 --- a/be/src/io/cache/file_cache_common.cpp +++ b/be/src/io/cache/file_cache_common.cpp @@ -26,6 +26,21 @@ namespace doris::io { +std::string file_cache_type_to_string(FileCacheType type) { + switch (type) { + case FileCacheType::INDEX: + return "INDEX"; + case FileCacheType::NORMAL: + return "NORMAL"; + case FileCacheType::DISPOSABLE: + return "DISPOSABLE"; + case FileCacheType::TTL: + return "TTL"; + default: + return "UNKNOWN"; + } +} + std::string FileCacheSettings::to_string() const { std::stringstream ss; ss << "capacity: " << capacity << ", max_file_block_size: " << max_file_block_size @@ -87,4 +102,41 @@ FileBlocksHolderPtr FileCacheAllocatorBuilder::allocate_cache_holder(size_t offs return std::make_unique(std::move(holder)); } +std::string FileCacheInfo::to_string() const { + std::stringstream ss; + ss << "Hash: " << hash.to_string() << "\n" + << "Expiration Time: " << expiration_time << "\n" + << "Offset: " << offset << "\n" + << "Cache Type: " << file_cache_type_to_string(cache_type) << "\n"; + return ss.str(); +} + +std::string InconsistencyType::to_string() const { + std::string result = "Inconsistency Reason: "; + if (type == NONE) { + result += "NONE"; + } else { + if (type & NOT_LOADED) { + result += "NOT_LOADED "; + } + if (type & MISSING_IN_STORAGE) { + result += "MISSING_IN_STORAGE "; + } + if (type & SIZE_INCONSISTENT) { + result += "SIZE_INCONSISTENT "; + } + if (type & CACHE_TYPE_INCONSISTENT) { + result += "CACHE_TYPE_INCONSISTENT "; + } + if (type & EXPIRATION_TIME_INCONSISTENT) { + result += "EXPIRATION_TIME_INCONSISTENT "; + } + if (type & TMP_FILE_EXPECT_DOWNLOADING_STATE) { + result += "TMP_FILE_EXPECT_DOWNLOADING_STATE"; + } + } + result += "\n"; + return result; +} + } // namespace doris::io diff --git a/be/src/io/cache/file_cache_common.h b/be/src/io/cache/file_cache_common.h index 0d700d9303191f4..5ba7826b143e97a 100644 --- a/be/src/io/cache/file_cache_common.h +++ b/be/src/io/cache/file_cache_common.h @@ -19,6 +19,9 @@ // and modified by Doris #pragma once +#include +#include + #include "io/io_common.h" #include "vec/common/uint128.h" @@ -39,6 +42,7 @@ enum FileCacheType { DISPOSABLE = 0, TTL = 3, }; +std::string file_cache_type_to_string(FileCacheType type); struct UInt128Wrapper { uint128_t value_; @@ -135,5 +139,48 @@ struct CacheContext { int64_t expiration_time {0}; bool is_cold_data {false}; }; +struct FileCacheInfo { + UInt128Wrapper hash {0}; + uint64 expiration_time {0}; + uint64_t size {0}; + size_t offset {0}; + bool is_tmp {false}; + FileCacheType cache_type {NORMAL}; + + std::string to_string() const; +}; + +class InconsistencyType { + uint32_t type; + +public: + enum : uint32_t { + // No anomaly + NONE = 0, + // Missing a block cache metadata in _files + NOT_LOADED = 1 << 0, + // A block cache is missing in storage + MISSING_IN_STORAGE = 1 << 1, + // Size of a block cache recorded in _files is inconsistent with the storage + SIZE_INCONSISTENT = 1 << 2, + // Cache type of a block cache recorded in _files is inconsistent with the storage + CACHE_TYPE_INCONSISTENT = 1 << 3, + // Expiration time of a block cache recorded in _files is inconsistent with the storage + EXPIRATION_TIME_INCONSISTENT = 1 << 4, + // File in storage has a _tmp suffix, but the state of block cache in _files is not set to downloading + TMP_FILE_EXPECT_DOWNLOADING_STATE = 1 << 5 + }; + InconsistencyType(uint32_t t = 0) : type(t) {} + operator uint32_t&() { return type; } + + std::string to_string() const; +}; + +struct InconsistencyContext { + // The infos in _files of BlockFileCache. + std::vector infos_in_manager; + std::vector infos_in_storage; + std::vector types; +}; } // namespace doris::io diff --git a/be/src/io/cache/file_cache_storage.h b/be/src/io/cache/file_cache_storage.h index 024e701c6fa08be..17253f8daf89826 100644 --- a/be/src/io/cache/file_cache_storage.h +++ b/be/src/io/cache/file_cache_storage.h @@ -17,6 +17,9 @@ #pragma once +#include + +#include "common/status.h" #include "io/cache/file_cache_common.h" #include "util/slice.h" @@ -67,6 +70,10 @@ class FileCacheStorage { virtual FileCacheStorageType get_type() = 0; // get local cached file virtual std::string get_local_file(const FileCacheKey& key) = 0; + virtual Status get_file_cache_infos(std::vector& infos, + std::lock_guard& cache_lock) const { + return Status::OK(); + }; }; } // namespace doris::io diff --git a/be/src/io/cache/fs_file_cache_storage.cpp b/be/src/io/cache/fs_file_cache_storage.cpp index cf1cd41a537abc0..78267b58507c400 100644 --- a/be/src/io/cache/fs_file_cache_storage.cpp +++ b/be/src/io/cache/fs_file_cache_storage.cpp @@ -17,15 +17,19 @@ #include "io/cache/fs_file_cache_storage.h" +#include + #include #include #include #include "common/logging.h" +#include "common/status.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" +#include "io/cache/file_cache_storage.h" #include "io/fs/file_reader_writer_fwd.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_reader.h" @@ -523,7 +527,7 @@ void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr) const std::error_code ec; std::filesystem::directory_iterator offset_it(key_it->path(), ec); if (ec) [[unlikely]] { - LOG(WARNING) << "filesystem error, failed to remove file, file=" << key_it->path() + LOG(WARNING) << "filesystem error, failed to list dir, dir=" << key_it->path() << " error=" << ec.message(); continue; } @@ -605,6 +609,69 @@ void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr) const TEST_SYNC_POINT_CALLBACK("BlockFileCache::TmpFile2"); } +Status FSFileCacheStorage::get_file_cache_infos(std::vector& infos, + std::lock_guard& cache_lock) const { + std::error_code ec; + std::filesystem::directory_iterator key_prefix_it {_cache_base_path, ec}; + if (ec) [[unlikely]] { + LOG(ERROR) << fmt::format("Failed to list dir {}, err={}", _cache_base_path, ec.message()); + return Status::InternalError("Failed to list dir {}, err={}", _cache_base_path, + ec.message()); + } + // Only supports version 2. For more details, refer to 'USE_CACHE_VERSION2'. + for (; key_prefix_it != std::filesystem::directory_iterator(); ++key_prefix_it) { + if (!key_prefix_it->is_directory()) { + // skip version file + continue; + } + if (key_prefix_it->path().filename().native().size() != KEY_PREFIX_LENGTH) { + LOG(WARNING) << "Unknown directory " << key_prefix_it->path().native(); + continue; + } + std::filesystem::directory_iterator key_it {key_prefix_it->path(), ec}; + if (ec) [[unlikely]] { + LOG(ERROR) << fmt::format("Failed to list dir {}, err={}", + key_prefix_it->path().filename().native(), ec.message()); + return Status::InternalError("Failed to list dir {}, err={}", + key_prefix_it->path().filename().native(), ec.message()); + } + for (; key_it != std::filesystem::directory_iterator(); ++key_it) { + auto key_with_suffix = key_it->path().filename().native(); + auto delim_pos = key_with_suffix.find('_'); + DCHECK(delim_pos != std::string::npos); + std::string key_str = key_with_suffix.substr(0, delim_pos); + std::string expiration_time_str = key_with_suffix.substr(delim_pos + 1); + long expiration_time = std::stoul(expiration_time_str); + auto hash = UInt128Wrapper(vectorized::unhex_uint(key_str.c_str())); + std::error_code ec; + std::filesystem::directory_iterator offset_it(key_it->path(), ec); + if (ec) [[unlikely]] { + LOG(ERROR) << fmt::format("Failed to list dir {}, err={}", + key_it->path().filename().native(), ec.message()); + return Status::InternalError("Failed to list dir {}, err={}", + key_it->path().filename().native(), ec.message()); + } + for (; offset_it != std::filesystem::directory_iterator(); ++offset_it) { + size_t size = offset_it->file_size(ec); + if (ec) [[unlikely]] { + LOG(ERROR) << fmt::format("Failed to get file size, file name {}, err={}", + key_it->path().filename().native(), ec.message()); + return Status::InternalError("Failed to get file size, file name {}, err={}", + key_it->path().filename().native(), ec.message()); + } + size_t offset = 0; + bool is_tmp = false; + FileCacheType cache_type = FileCacheType::NORMAL; + RETURN_IF_ERROR(this->parse_filename_suffix_to_cache_type( + fs, offset_it->path().filename().native(), expiration_time, size, &offset, + &is_tmp, &cache_type)); + infos.emplace_back(hash, expiration_time, size, offset, is_tmp, cache_type); + } + } + } + return Status::OK(); +} + void FSFileCacheStorage::load_blocks_directly_unlocked(BlockFileCache* mgr, const FileCacheKey& key, std::lock_guard& cache_lock) { // async load, can't find key, need to check exist. diff --git a/be/src/io/cache/fs_file_cache_storage.h b/be/src/io/cache/fs_file_cache_storage.h index 8a97aa109ad7411..e8a0f0c32e9e95c 100644 --- a/be/src/io/cache/fs_file_cache_storage.h +++ b/be/src/io/cache/fs_file_cache_storage.h @@ -105,6 +105,9 @@ class FSFileCacheStorage : public FileCacheStorage { [[nodiscard]] std::vector get_path_in_local_cache_all_candidates( const std::string& dir, size_t offset); + Status get_file_cache_infos(std::vector& infos, + std::lock_guard& cache_lock) const override; + std::string _cache_base_path; std::thread _cache_background_load_thread; const std::shared_ptr& fs = global_local_filesystem(); diff --git a/be/test/io/cache/block_file_cache_test.cpp b/be/test/io/cache/block_file_cache_test.cpp index 11e99a4805286f7..4ff8a7b7816acbe 100644 --- a/be/test/io/cache/block_file_cache_test.cpp +++ b/be/test/io/cache/block_file_cache_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #if defined(__APPLE__) #include @@ -5284,6 +5285,155 @@ TEST_F(BlockFileCacheTest, file_cache_path_storage_parse) { } } +TEST_F(BlockFileCacheTest, check_fs_file_cache_consistency) { + if (fs::exists(cache_base_path)) { + fs::remove_all(cache_base_path); + } + fs::create_directories(cache_base_path); + TUniqueId query_id; + query_id.hi = 1; + query_id.lo = 1; + io::FileCacheSettings settings; + settings.query_queue_size = 30; + settings.query_queue_elements = 5; + settings.index_queue_size = 30; + settings.index_queue_elements = 5; + settings.disposable_queue_size = 30; + settings.disposable_queue_elements = 5; + settings.capacity = 90; + settings.max_file_block_size = 30; + settings.max_query_cache_size = 30; + auto key1 = io::BlockFileCache::hash("key1"); + auto key2 = io::BlockFileCache::hash("key2"); + + io::BlockFileCache mgr(cache_base_path, settings); + ASSERT_TRUE(mgr.initialize()); + for (int i = 0; i < 100; i++) { + if (mgr.get_async_open_success()) { + break; + }; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + io::CacheContext cache_context; + cache_context.cache_type = io::FileCacheType::TTL; + cache_context.query_id = query_id; + cache_context.expiration_time = 0; + { + cache_context.cache_type = io::FileCacheType::NORMAL; + auto holder = mgr.get_or_set(key1, 0, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(0, 8), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(0, 8), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + std::vector result; + Status status = mgr.report_file_cache_inconsistency(result); + ASSERT_TRUE(result.empty()); + } + + { + auto holder = mgr.get_or_set(key1, 10, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(10, 18), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(10, 18), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + mgr._files[key1].erase(10); + } + + { + auto holder = mgr.get_or_set(key1, 20, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(20, 28), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(20, 28), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + auto* fs_file_cache_storage = dynamic_cast(mgr._storage.get()); + std::string dir_path = fs_file_cache_storage->get_path_in_local_cache(key1, 0); + fs::path block_file_path = std::filesystem::path(dir_path) / "20"; + fs::remove(block_file_path); + } + + { + auto holder = mgr.get_or_set(key1, 30, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(30, 38), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(30, 38), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + auto* fs_file_cache_storage = dynamic_cast(mgr._storage.get()); + std::string dir_path = fs_file_cache_storage->get_path_in_local_cache(key1, 0); + fs::path block_file_path = std::filesystem::path(dir_path) / "30"; + std::string data = "This is a test message."; + std::ofstream out_file(block_file_path, std::ios::out | std::ios::app); + out_file << data; + out_file.close(); + } + + { + auto holder = mgr.get_or_set(key1, 40, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(40, 48), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(40, 48), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + blocks[0]->_key.meta.type = io::FileCacheType::INDEX; + } + + int64_t expiration_time = UnixSeconds() + 120; + { + cache_context.cache_type = FileCacheType::TTL; + cache_context.expiration_time = expiration_time; + auto holder = mgr.get_or_set(key2, 0, 9, cache_context); + auto blocks = fromHolder(holder); + ASSERT_EQ(blocks.size(), 1); + assert_range(1, blocks[0], io::FileBlock::Range(0, 8), io::FileBlock::State::EMPTY); + ASSERT_TRUE(blocks[0]->get_or_set_downloader() == io::FileBlock::get_caller_id()); + assert_range(2, blocks[0], io::FileBlock::Range(0, 8), io::FileBlock::State::DOWNLOADING); + download(blocks[0]); + blocks[0]->_key.meta.expiration_time = 0; + } + std::vector result; + Status status = mgr.report_file_cache_inconsistency(result); + std::vector expected_result = { + "File cahce info in manager:\nHash: " + "62434304659ae12df53386481113dfe1\nExpiration Time: 0\nOffset: 0\nCache Type: " + "TTL\nFile cahce info in storage:\nHash: " + "62434304659ae12df53386481113dfe1\nExpiration Time: " + + std::to_string(expiration_time) + + "\nOffset: 0\nCache Type: " + "TTL\nInconsistency Reason: EXPIRATION_TIME_INCONSISTENT \n\n", + "File cahce info in manager:\nHash: " + "00000000000000000000000000000000\nExpiration Time: 0\nOffset: 0\nCache Type: " + "NORMAL\nFile cahce info in storage:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 10\nCache Type: " + "NORMAL\nInconsistency Reason: NOT_LOADED \n\n", + "File cahce info in manager:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 30\nCache Type: " + "NORMAL\nFile cahce info in storage:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 30\nCache Type: " + "NORMAL\nInconsistency Reason: SIZE_INCONSISTENT \n\n", + "File cahce info in manager:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 40\nCache Type: " + "INDEX\nFile cahce info in storage:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 40\nCache Type: " + "NORMAL\nInconsistency Reason: CACHE_TYPE_INCONSISTENT \n\n", + "File cahce info in manager:\nHash: " + "f36131fb4ba563c17e727cd0cdd63689\nExpiration Time: 0\nOffset: 20\nCache Type: " + "NORMAL\nFile cahce info in storage:\nHash: " + "00000000000000000000000000000000\nExpiration Time: 0\nOffset: 0\nCache Type: " + "NORMAL\nInconsistency Reason: MISSING_IN_STORAGE \n\n"}; + ASSERT_EQ(result.size(), expected_result.size()); + for (size_t i = 0; i < result.size(); i++) { + ASSERT_EQ(expected_result[i], result[i]); + } +} + TEST_F(BlockFileCacheTest, populate_empty_cache_with_disposable) { if (fs::exists(cache_base_path)) { fs::remove_all(cache_base_path);