From 97a8ffdf05dee72abc198964a1f0a7a3ae4b9742 Mon Sep 17 00:00:00 2001 From: Yueh-Hsuan Chiang Date: Wed, 14 Feb 2024 16:27:26 -0700 Subject: [PATCH 1/2] [TieredStorage] Handles reduced-offset and offset conversion --- accounts-db/src/account_info.rs | 4 +-- accounts-db/src/accounts_file.rs | 48 ++++++++++++++++++++++++-------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/accounts-db/src/account_info.rs b/accounts-db/src/account_info.rs index cbec32be6499ef..5e4cb01b0c8af4 100644 --- a/accounts-db/src/account_info.rs +++ b/accounts-db/src/account_info.rs @@ -156,7 +156,7 @@ impl AccountInfo { } } - fn get_reduced_offset(offset: usize) -> OffsetReduced { + pub fn get_reduced_offset(offset: usize) -> OffsetReduced { (offset / ALIGN_BOUNDARY_OFFSET) as OffsetReduced } @@ -174,7 +174,7 @@ impl AccountInfo { ) } - fn reduced_offset_to_offset(reduced_offset: OffsetReduced) -> Offset { + pub fn reduced_offset_to_offset(reduced_offset: OffsetReduced) -> Offset { (reduced_offset as Offset) * ALIGN_BOUNDARY_OFFSET } diff --git a/accounts-db/src/accounts_file.rs b/accounts-db/src/accounts_file.rs index 6371be6083cc84..a50b5ad42cbf4e 100644 --- a/accounts-db/src/accounts_file.rs +++ b/accounts-db/src/accounts_file.rs @@ -1,5 +1,6 @@ use { crate::{ + account_info::AccountInfo, account_storage::meta::{ StorableAccountsWithHashesAndWriteVersions, StoredAccountInfo, StoredAccountMeta, }, @@ -120,14 +121,19 @@ impl AccountsFile { /// Return (account metadata, next_index) pair for the account at the /// specified `index` if any. Otherwise return None. Also return the /// index of the next entry. - pub fn get_account(&self, index: usize) -> Option<(StoredAccountMeta<'_>, usize)> { + pub fn get_account(&self, offset: usize) -> Option<(StoredAccountMeta<'_>, usize)> { match self { - Self::AppendVec(av) => av.get_account(index), + Self::AppendVec(av) => av.get_account(offset), + // Note: The conversion here is needed as the AccountsDB currently + // assumes all offsets are multiple of 8 while TieredStorage uses + // IndexOffset that is equivalent to AccountInfo::reduced_offset. Self::TieredStorage(ts) => ts .reader()? - .get_account(IndexOffset(index as u32)) + .get_account(IndexOffset(AccountInfo::get_reduced_offset(offset))) .ok()? - .map(|(metas, index_offset)| (metas, index_offset.0 as usize)), + .map(|(metas, index_offset)| { + (metas, AccountInfo::reduced_offset_to_offset(index_offset.0)) + }), } } @@ -138,11 +144,17 @@ impl AccountsFile { ) -> std::result::Result { match self { Self::AppendVec(av) => av.account_matches_owners(offset, owners), + // Note: The conversion here is needed as the AccountsDB currently + // assumes all offsets are multiple of 8 while TieredStorage uses + // IndexOffset that is equivalent to AccountInfo::reduced_offset. Self::TieredStorage(ts) => { let Some(reader) = ts.reader() else { return Err(MatchAccountOwnerError::UnableToLoad); }; - reader.account_matches_owners(IndexOffset(offset as u32), owners) + reader.account_matches_owners( + IndexOffset(AccountInfo::get_reduced_offset(offset)), + owners, + ) } } } @@ -164,9 +176,16 @@ impl AccountsFile { pub fn accounts(&self, offset: usize) -> Vec { match self { Self::AppendVec(av) => av.accounts(offset), + // Note: The conversion here is needed as the AccountsDB currently + // assumes all offsets are multiple of 8 while TieredStorage uses + // IndexOffset that is equivalent to AccountInfo::reduced_offset. Self::TieredStorage(ts) => ts .reader() - .and_then(|reader| reader.accounts(IndexOffset(offset as u32)).ok()) + .and_then(|reader| { + reader + .accounts(IndexOffset(AccountInfo::get_reduced_offset(offset))) + .ok() + }) .unwrap_or_default(), } } @@ -191,11 +210,18 @@ impl AccountsFile { ) -> Option> { match self { Self::AppendVec(av) => av.append_accounts(accounts, skip), - // Currently we only support HOT_FORMAT. If we later want to use - // a different format, then we will need a way to pass-in it. - // TODO: consider adding function like write_accounts_to_hot_storage() or something - // to hide implementation detail. - Self::TieredStorage(ts) => ts.write_accounts(accounts, skip, &HOT_FORMAT).ok(), + // Note: The conversion here is needed as the AccountsDB currently + // assumes all offsets are multiple of 8 while TieredStorage uses + // IndexOffset that is equivalent to AccountInfo::reduced_offset. + Self::TieredStorage(ts) => ts + .write_accounts(accounts, skip, &HOT_FORMAT) + .map(|mut infos| { + infos.iter_mut().for_each(|info| { + info.offset = AccountInfo::reduced_offset_to_offset(info.offset as u32); + }); + infos + }) + .ok(), } } } From 2aa876c642a221658f93e45d66f6a76aef99ad2b Mon Sep 17 00:00:00 2001 From: jeff washington Date: Fri, 29 Mar 2024 17:02:46 -0500 Subject: [PATCH 2/2] fix comment --- accounts-db/src/accounts_file.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts-db/src/accounts_file.rs b/accounts-db/src/accounts_file.rs index a50b5ad42cbf4e..004f14f5e855fa 100644 --- a/accounts-db/src/accounts_file.rs +++ b/accounts-db/src/accounts_file.rs @@ -119,7 +119,7 @@ impl AccountsFile { } /// Return (account metadata, next_index) pair for the account at the - /// specified `index` if any. Otherwise return None. Also return the + /// specified `offset` if any. Otherwise return None. Also return the /// index of the next entry. pub fn get_account(&self, offset: usize) -> Option<(StoredAccountMeta<'_>, usize)> { match self {