Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix: exception in OptimizeIndex #2499

Merged
merged 2 commits into from
Jan 27, 2025
Merged

Fix: exception in OptimizeIndex #2499

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56f4e7f
update debug info
yangzq50 Jan 27, 2025
c65618c
update merge ft index
yangzq50 Jan 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions src/storage/meta/entry/segment_index_entry.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -658,13 +658,15 @@ void SegmentIndexEntry::GetChunkIndexEntries(Vector<SharedPtr<ChunkIndexEntry>>
for (SizeT i = 0; i < num; i++) {
auto &chunk_index_entry = chunk_index_entries_[i];
bool add = chunk_index_entry->CheckVisible(txn);
LOG_INFO(fmt::format("GetChunkIndexEntries, CheckVisible ret: {}, chunk_id: {}, deprecate ts: {}, txn_id: {}, base_rowid: {}, row_count: {}",
add,
chunk_index_entry->chunk_id_,
chunk_index_entry->deprecate_ts_.load(),
txn_id_str,
chunk_index_entry->base_rowid_.ToString(),
chunk_index_entry->row_count_));
LOG_INFO(fmt::format(
"GetChunkIndexEntries, CheckVisible ret: {}, chunk_id: {}, deprecate ts: {}, txn_id: {}, commit_ts: {}, base_rowid: {}, row_count: {}",
add,
chunk_index_entry->chunk_id_,
chunk_index_entry->deprecate_ts_.load(),
chunk_index_entry->txn_id_,
chunk_index_entry->commit_ts_.load(),
chunk_index_entry->base_rowid_.ToString(),
chunk_index_entry->row_count_));
if (add) {
chunk_index_entries.push_back(chunk_index_entry);
}
Expand Down
43 changes: 26 additions & 17 deletions src/storage/meta/entry/table_entry.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -970,11 +970,10 @@ void TableEntry::OptimizeIndex(Txn *txn) {
opt_success = true;
continue;
}
add_segment_optimizing();
String msg = fmt::format("merging {}", index_name);
Vector<String> base_names;
Vector<RowID> base_rowids;
RowID base_rowid = chunk_index_entries[0]->base_rowid_;
const RowID base_rowid = chunk_index_entries[0]->base_rowid_;
u32 total_row_count = 0;

for (SizeT i = 0; i < chunk_index_entries.size(); i++) {
Expand All @@ -986,26 +985,36 @@ void TableEntry::OptimizeIndex(Txn *txn) {
chunk_index_entry->row_count_));
}

for (SizeT i = 0; i < chunk_index_entries.size(); i++) {
auto &chunk_index_entry = chunk_index_entries[i];
msg += " " + chunk_index_entry->base_name_;

if (chunk_index_entry->base_rowid_ != chunk_index_entries[0]->base_rowid_ + total_row_count) {
String error_msg = fmt::format("{}... chunk_index_entry {} base_rowid expects to be {:016x}",
msg,
chunk_index_entry->base_name_,
(chunk_index_entries[0]->base_rowid_ + total_row_count).ToUint64());

// merging text_index ft_0000000000000000_8000 ft_000000000000e000... chunk_index_entry
// ft_000000000000e000 base_rowid expects to be
// 0000000000008000@src/storage/meta/entry/table_entry.cpp:955

UnrecoverableError(error_msg);
for (auto it = chunk_index_entries.begin(); it != chunk_index_entries.end(); ++it) {
auto &chunk_index_entry = *it;
if (const RowID expect_base_row_id = base_rowid + total_row_count; chunk_index_entry->base_rowid_ > expect_base_row_id) {
msg += fmt::format(" stop at gap to chunk {}, expect_base_row_id: {:016x}, base_row_id: {:016x}",
chunk_index_entry->base_name_,
expect_base_row_id.ToUint64(),
chunk_index_entry->base_rowid_.ToUint64());
chunk_index_entries.erase(it, chunk_index_entries.end());
break;
} else if (chunk_index_entry->base_rowid_ < expect_base_row_id) {
msg += fmt::format(" found overlap to chunk {}, expect_base_row_id: {:016x}, base_row_id: {:016x}",
chunk_index_entry->base_name_,
expect_base_row_id.ToUint64(),
chunk_index_entry->base_rowid_.ToUint64());
UnrecoverableError(msg);
}
msg += " " + chunk_index_entry->base_name_;
base_names.push_back(chunk_index_entry->base_name_);
base_rowids.push_back(chunk_index_entry->base_rowid_);
total_row_count += chunk_index_entry->row_count_;
}

if (chunk_index_entries.size() <= 1) {
msg += fmt::format(" skip merge due to only {} chunk", chunk_index_entries.size());
LOG_INFO(msg);
opt_success = true;
continue;
}

add_segment_optimizing();
String dst_base_name = fmt::format("ft_{:016x}_{:x}", base_rowid.ToUint64(), total_row_count);
msg += " -> " + dst_base_name;
LOG_INFO(msg);
Expand Down