From 56f4e7ff88035c1b2cc81c28bbc22f613f7d003b Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 27 Jan 2025 11:12:27 +0800 Subject: [PATCH 1/2] update debug info --- src/storage/meta/entry/segment_index_entry.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/storage/meta/entry/segment_index_entry.cpp b/src/storage/meta/entry/segment_index_entry.cpp index 5e3cfd5ab8..43a6d05798 100644 --- a/src/storage/meta/entry/segment_index_entry.cpp +++ b/src/storage/meta/entry/segment_index_entry.cpp @@ -658,13 +658,15 @@ void SegmentIndexEntry::GetChunkIndexEntries(Vector> for (SizeT i = 0; i < num; i++) { auto &chunk_index_entry = chunk_index_entries_[i]; bool add = chunk_index_entry->CheckVisible(txn); - LOG_INFO(fmt::format("GetChunkIndexEntries, CheckVisible ret: {}, chunk_id: {}, deprecate ts: {}, txn_id: {}, base_rowid: {}, row_count: {}", - add, - chunk_index_entry->chunk_id_, - chunk_index_entry->deprecate_ts_.load(), - txn_id_str, - chunk_index_entry->base_rowid_.ToString(), - chunk_index_entry->row_count_)); + LOG_INFO(fmt::format( + "GetChunkIndexEntries, CheckVisible ret: {}, chunk_id: {}, deprecate ts: {}, txn_id: {}, commit_ts: {}, base_rowid: {}, row_count: {}", + add, + chunk_index_entry->chunk_id_, + chunk_index_entry->deprecate_ts_.load(), + chunk_index_entry->txn_id_, + chunk_index_entry->commit_ts_.load(), + chunk_index_entry->base_rowid_.ToString(), + chunk_index_entry->row_count_)); if (add) { chunk_index_entries.push_back(chunk_index_entry); } From c65618c5f8dc9f6d464c982472f69295b782853c Mon Sep 17 00:00:00 2001 From: yzq <58433399+yangzq50@users.noreply.github.com> Date: Mon, 27 Jan 2025 12:08:13 +0800 Subject: [PATCH 2/2] update merge ft index --- src/storage/meta/entry/table_entry.cpp | 43 ++++++++++++++++---------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/src/storage/meta/entry/table_entry.cpp b/src/storage/meta/entry/table_entry.cpp index 935aa3d4cc..a1bb51ebec 100644 --- a/src/storage/meta/entry/table_entry.cpp +++ b/src/storage/meta/entry/table_entry.cpp @@ -970,11 +970,10 @@ void TableEntry::OptimizeIndex(Txn *txn) { opt_success = true; continue; } - add_segment_optimizing(); String msg = fmt::format("merging {}", index_name); Vector base_names; Vector base_rowids; - RowID base_rowid = chunk_index_entries[0]->base_rowid_; + const RowID base_rowid = chunk_index_entries[0]->base_rowid_; u32 total_row_count = 0; for (SizeT i = 0; i < chunk_index_entries.size(); i++) { @@ -986,26 +985,36 @@ void TableEntry::OptimizeIndex(Txn *txn) { chunk_index_entry->row_count_)); } - for (SizeT i = 0; i < chunk_index_entries.size(); i++) { - auto &chunk_index_entry = chunk_index_entries[i]; - msg += " " + chunk_index_entry->base_name_; - - if (chunk_index_entry->base_rowid_ != chunk_index_entries[0]->base_rowid_ + total_row_count) { - String error_msg = fmt::format("{}... chunk_index_entry {} base_rowid expects to be {:016x}", - msg, - chunk_index_entry->base_name_, - (chunk_index_entries[0]->base_rowid_ + total_row_count).ToUint64()); - - // merging text_index ft_0000000000000000_8000 ft_000000000000e000... chunk_index_entry - // ft_000000000000e000 base_rowid expects to be - // 0000000000008000@src/storage/meta/entry/table_entry.cpp:955 - - UnrecoverableError(error_msg); + for (auto it = chunk_index_entries.begin(); it != chunk_index_entries.end(); ++it) { + auto &chunk_index_entry = *it; + if (const RowID expect_base_row_id = base_rowid + total_row_count; chunk_index_entry->base_rowid_ > expect_base_row_id) { + msg += fmt::format(" stop at gap to chunk {}, expect_base_row_id: {:016x}, base_row_id: {:016x}", + chunk_index_entry->base_name_, + expect_base_row_id.ToUint64(), + chunk_index_entry->base_rowid_.ToUint64()); + chunk_index_entries.erase(it, chunk_index_entries.end()); + break; + } else if (chunk_index_entry->base_rowid_ < expect_base_row_id) { + msg += fmt::format(" found overlap to chunk {}, expect_base_row_id: {:016x}, base_row_id: {:016x}", + chunk_index_entry->base_name_, + expect_base_row_id.ToUint64(), + chunk_index_entry->base_rowid_.ToUint64()); + UnrecoverableError(msg); } + msg += " " + chunk_index_entry->base_name_; base_names.push_back(chunk_index_entry->base_name_); base_rowids.push_back(chunk_index_entry->base_rowid_); total_row_count += chunk_index_entry->row_count_; } + + if (chunk_index_entries.size() <= 1) { + msg += fmt::format(" skip merge due to only {} chunk", chunk_index_entries.size()); + LOG_INFO(msg); + opt_success = true; + continue; + } + + add_segment_optimizing(); String dst_base_name = fmt::format("ft_{:016x}_{:x}", base_rowid.ToUint64(), total_row_count); msg += " -> " + dst_base_name; LOG_INFO(msg);