Tweak ancient packing algorithm

accounts-db/src/accounts_db.rs

@@ -1479,7 +1479,7 @@ pub struct AccountsDb {
     /// Set of stores which are recently rooted or had accounts removed
     /// such that potentially a 0-lamport account update could be present which
     /// means we can remove the account from the index entirely.
-    dirty_stores: DashMap<Slot, Arc<AccountStorageEntry>>,
+    pub(crate) dirty_stores: DashMap<Slot, Arc<AccountStorageEntry>>,
 
     /// Zero-lamport accounts that are *not* purged during clean because they need to stay alive
     /// for incremental snapshot support.
@@ -1521,6 +1521,11 @@ pub struct AccountsDb {
 
     /// The latest full snapshot slot dictates how to handle zero lamport accounts
     latest_full_snapshot_slot: SeqLock<Option<Slot>>,
+
+    /// These are the ancient storages that could be valuable to shrink.
+    /// sorted by largest dead bytes to smallest
+    /// Members are Slot and capacity. If capacity is smaller, then that means the storage was already shrunk.
+    pub(crate) best_ancient_slots_to_shrink: RwLock<Vec<(Slot, u64)>>,
 }
 
 #[derive(Debug, Default)]
@@ -2491,6 +2496,7 @@ impl AccountsDb {
         const ACCOUNTS_STACK_SIZE: usize = 8 * 1024 * 1024;
 
         AccountsDb {
+            best_ancient_slots_to_shrink: RwLock::default(),
             create_ancient_storage: CreateAncientStorage::default(),
             verify_accounts_hash_in_bg: VerifyAccountsHashInBackground::default(),
             active_stats: ActiveStats::default(),
@@ -5059,7 +5065,7 @@ impl AccountsDb {
         let shrink_candidates_slots =
             std::mem::take(&mut *self.shrink_candidate_slots.lock().unwrap());
 
-        let (shrink_slots, shrink_slots_next_batch) = {
+        let (mut shrink_slots, shrink_slots_next_batch) = {
             if let AccountShrinkThreshold::TotalSpace { shrink_ratio } = self.shrink_ratio {
                 let (shrink_slots, shrink_slots_next_batch) =
                     self.select_candidates_by_total_usage(&shrink_candidates_slots, shrink_ratio);
@@ -5080,6 +5086,31 @@ impl AccountsDb {
             }
         };
 
+        let mut limit = 2;
+        if shrink_slots.len() >= limit {
+            limit = shrink_slots.len() + 1;
+        }
+        let mut ancients = self.best_ancient_slots_to_shrink.write().unwrap();
+        for (slot, capacity) in ancients.iter_mut() {
+            if *capacity == 0 || shrink_slots.contains(slot) {
+                // already dealt with
+                continue;
+            }
+            // we will be done processing this suggestion no matter what
+            if let Some(store) = self.storage.get_slot_storage_entry(*slot) {
+                if *capacity != store.capacity() || !Self::is_candidate_for_shrink(self, &store) {
+                    *capacity = 0;
+                    // ignore this one
+                    continue;
+                }
+                *capacity = 0;
+                shrink_slots.insert(*slot, store);
+
+                if shrink_slots.len() >= limit {
+                    break;
+                }
+            }
+        }
         if shrink_slots.is_empty()
             && shrink_slots_next_batch
                 .as_ref()
@@ -8051,7 +8082,7 @@ impl AccountsDb {
         true
     }
 
-    fn is_candidate_for_shrink(&self, store: &AccountStorageEntry) -> bool {
+    pub(crate) fn is_candidate_for_shrink(&self, store: &AccountStorageEntry) -> bool {
         // appended ancient append vecs should not be shrunk by the normal shrink codepath.
         // It is not possible to identify ancient append vecs when we pack, so no check for ancient when we are not appending.
         let total_bytes = if self.create_ancient_storage == CreateAncientStorage::Append