Skip to content

Commit

Permalink
next attempt at packing algorithm
Browse files Browse the repository at this point in the history
  • Loading branch information
@jeffwashington
jeffwashington committed Aug 29, 2024
1 parent ae93256 commit 4f4fa9f
Show file tree
Hide file tree
Showing 2 changed files with 66 additions and 27 deletions.
44 changes: 27 additions & 17 deletions accounts-db/src/accounts_db.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -585,6 +585,8 @@ pub struct AccountsAddRootTiming {
pub store_us: u64,
}

/// if negative, this many accounts older than # slots in epoch are still treated as modern (ie. non-ancient).
/// Slots older than # slots in epoch - this # are then treated as ancient and subject to packing.
const ANCIENT_APPEND_VEC_DEFAULT_OFFSET: Option<i64> = Some(-10_000);

#[derive(Debug, Default, Clone)]
Expand Down Expand Up @@ -1478,7 +1480,7 @@ pub struct AccountsDb {
/// Set of stores which are recently rooted or had accounts removed
/// such that potentially a 0-lamport account update could be present which
/// means we can remove the account from the index entirely.
dirty_stores: DashMap<Slot, Arc<AccountStorageEntry>>,
pub(crate) dirty_stores: DashMap<Slot, Arc<AccountStorageEntry>>,

/// Zero-lamport accounts that are *not* purged during clean because they need to stay alive
/// for incremental snapshot support.
Expand Down Expand Up @@ -1990,6 +1992,7 @@ pub(crate) struct ShrinkAncientStats {
pub(crate) many_refs_old_alive: AtomicU64,
pub(crate) slots_eligible_to_shrink: AtomicU64,
pub(crate) total_dead_bytes: AtomicU64,
pub(crate) ideal_storage_size: AtomicU64,
}

#[derive(Debug, Default)]
Expand Down Expand Up @@ -2038,6 +2041,7 @@ pub struct ShrinkStats {
accounts_loaded: AtomicU64,
purged_zero_lamports: AtomicU64,
accounts_not_found_in_index: AtomicU64,
shrinking_ancient: AtomicU64,
}

impl ShrinkStats {
Expand Down Expand Up @@ -2151,6 +2155,11 @@ impl ShrinkStats {
self.purged_zero_lamports.swap(0, Ordering::Relaxed),
i64
),
(
"shrinking_ancient",
self.shrinking_ancient.swap(0, Ordering::Relaxed),
i64
),
(
"accounts_not_found_in_index",
self.accounts_not_found_in_index.swap(0, Ordering::Relaxed),
Expand Down Expand Up @@ -2315,6 +2324,11 @@ impl ShrinkAncientStats {
self.slots_eligible_to_shrink.swap(0, Ordering::Relaxed),
i64
),
(
"ideal_storage_size",
self.ideal_storage_size.swap(0, Ordering::Relaxed),
i64
),
(
"total_dead_bytes",
self.total_dead_bytes.swap(0, Ordering::Relaxed),
Expand Down Expand Up @@ -4542,7 +4556,6 @@ impl AccountsDb {
&self,
shrink_slots: &ShrinkCandidates,
shrink_ratio: f64,
oldest_non_ancient_slot: Option<Slot>,
) -> (IntMap<Slot, Arc<AccountStorageEntry>>, ShrinkCandidates) {
struct StoreUsageInfo {
slot: Slot,
Expand All @@ -4556,13 +4569,6 @@ impl AccountsDb {
let mut total_bytes: u64 = 0;
let mut total_candidate_stores: usize = 0;
for slot in shrink_slots {
if oldest_non_ancient_slot
.map(|oldest_non_ancient_slot| slot < &oldest_non_ancient_slot)
.unwrap_or_default()
{
// this slot will be 'shrunk' by ancient code
continue;
}
let Some(store) = self.storage.get_slot_storage_entry(*slot) else {
continue;
};
Expand Down Expand Up @@ -5033,13 +5039,8 @@ impl AccountsDb {

let (shrink_slots, shrink_slots_next_batch) = {
if let AccountShrinkThreshold::TotalSpace { shrink_ratio } = self.shrink_ratio {
let (shrink_slots, shrink_slots_next_batch) = self
.select_candidates_by_total_usage(
&shrink_candidates_slots,
shrink_ratio,
self.ancient_append_vec_offset
.map(|_| oldest_non_ancient_slot),
);
let (shrink_slots, shrink_slots_next_batch) =
self.select_candidates_by_total_usage(&shrink_candidates_slots, shrink_ratio);
(shrink_slots, Some(shrink_slots_next_batch))
} else {
(
Expand Down Expand Up @@ -5076,6 +5077,15 @@ impl AccountsDb {
shrink_slots
.into_par_iter()
.for_each(|(slot, slot_shrink_candidate)| {
if self
.ancient_append_vec_offset
.map(|_| slot < oldest_non_ancient_slot)
.unwrap_or_default()
{
self.shrink_stats
.shrinking_ancient
.fetch_add(1, Ordering::Relaxed);
}
let mut measure = Measure::start("shrink_candidate_slots-ms");
self.do_shrink_slot_store(slot, &slot_shrink_candidate);
measure.stop();
Expand Down Expand Up @@ -8024,7 +8034,7 @@ impl AccountsDb {
true
}

fn is_candidate_for_shrink(&self, store: &AccountStorageEntry) -> bool {
pub(crate) fn is_candidate_for_shrink(&self, store: &AccountStorageEntry) -> bool {
// appended ancient append vecs should not be shrunk by the normal shrink codepath.
// It is not possible to identify ancient append vecs when we pack, so no check for ancient when we are not appending.
let total_bytes = if self.create_ancient_storage == CreateAncientStorage::Append
Expand Down
49 changes: 39 additions & 10 deletions accounts-db/src/ancient_append_vecs.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,19 @@ impl AncientSlotInfos {
can_randomly_shrink: bool,
ideal_size: NonZeroU64,
is_high_slot: bool,
db: &AccountsDb,
) -> bool {
let mut was_randomly_shrunk = false;
let alive_bytes = storage.alive_bytes() as u64;
if alive_bytes > 0 {
let capacity = storage.accounts.capacity();
let should_shrink = if capacity > 0 {
let alive_ratio = alive_bytes * 100 / capacity;
alive_ratio < 90
let candidate_for_shrink = db.is_candidate_for_shrink(&storage);
if candidate_for_shrink {
// shrink should run on this
db.shrink_candidate_slots.lock().unwrap().insert(slot);
}
candidate_for_shrink
|| if can_randomly_shrink && thread_rng().gen_range(0..10000) == 0 {
was_randomly_shrunk = true;
true
Expand Down Expand Up @@ -329,11 +334,20 @@ impl AccountsDb {
sorted_slots: Vec<Slot>,
can_randomly_shrink: bool,
) {
/*
// maybe we want this to be a different tuning parameter passed on cli?
let max_ancient_slots = if let Some(offset) = self.ancient_append_vec_offset {
offset.unsigned_abs().min(10_000) as usize
} else {
return;
};
*/
let max_ancient_slots = 4400; // this gives us approx 130M ideal size for testing against mnb now

let tuning = PackedAncientStorageTuning {
// only allow 10k slots old enough to be ancient
max_ancient_slots: 10_000,
// re-combine/shrink 55% of the data savings this pass
percent_of_alive_shrunk_data: 55,
max_ancient_slots,
// don't re-pack anything just to shrink. Shrink will handle these old storages.
percent_of_alive_shrunk_data: 0,
ideal_storage_size: NonZeroU64::new(get_ancient_append_vec_capacity()).unwrap(),
can_randomly_shrink,
max_resulting_storages: NonZeroU64::new(10).unwrap(),
Expand Down Expand Up @@ -391,13 +405,13 @@ impl AccountsDb {
fn combine_ancient_slots_packed_internal(
&self,
sorted_slots: Vec<Slot>,
tuning: PackedAncientStorageTuning,
mut tuning: PackedAncientStorageTuning,
metrics: &mut ShrinkStatsSub,
) {
self.shrink_ancient_stats
.slots_considered
.fetch_add(sorted_slots.len() as u64, Ordering::Relaxed);
let ancient_slot_infos = self.collect_sort_filter_ancient_slots(sorted_slots, &tuning);
let ancient_slot_infos = self.collect_sort_filter_ancient_slots(sorted_slots, &mut tuning);

if ancient_slot_infos.all_infos.is_empty() {
return; // nothing to do
Expand Down Expand Up @@ -511,12 +525,13 @@ impl AccountsDb {
fn collect_sort_filter_ancient_slots(
&self,
slots: Vec<Slot>,
tuning: &PackedAncientStorageTuning,
tuning: &mut PackedAncientStorageTuning,
) -> AncientSlotInfos {
let mut ancient_slot_infos = self.calc_ancient_slot_info(
slots,
tuning.can_randomly_shrink,
tuning.ideal_storage_size,
tuning,
);

ancient_slot_infos.filter_ancient_slots(tuning, &self.shrink_ancient_stats);
Expand Down Expand Up @@ -557,6 +572,7 @@ impl AccountsDb {
slots: Vec<Slot>,
can_randomly_shrink: bool,
ideal_size: NonZeroU64,
tuning: &mut PackedAncientStorageTuning,
) -> AncientSlotInfos {
let len = slots.len();
let mut infos = AncientSlotInfos {
Expand All @@ -578,19 +594,32 @@ impl AccountsDb {
can_randomly_shrink,
ideal_size,
is_high_slot(*slot),
&self,
) {
randoms += 1;
}
}
}
let mut total_dead_bytes = 0;
let mut total_alive_bytes = 0;
let should_shrink_count = infos
.all_infos
.iter()
.filter(|info| info.should_shrink)
.map(|info| total_dead_bytes += info.capacity.saturating_sub(info.alive_bytes))
.map(|info| {
total_dead_bytes += info.capacity.saturating_sub(info.alive_bytes);
total_alive_bytes += info.alive_bytes;
})
.count()
.saturating_sub(randoms as usize);
// ideal storage size is total alive bytes of ancient storages / half of max ancient slots
tuning.ideal_storage_size = NonZeroU64::new(
(infos.total_alive_bytes.0 / tuning.max_ancient_slots.max(1) as u64 / 2).max(1_000_000),
)
.unwrap();
self.shrink_ancient_stats
.ideal_storage_size
.store(tuning.ideal_storage_size.into(), Ordering::Relaxed);
self.shrink_ancient_stats
.slots_eligible_to_shrink
.fetch_add(should_shrink_count as u64, Ordering::Relaxed);
Expand Down

0 comments on commit 4f4fa9f

Please sign in to comment.