Runtime: add RentCollectorWithMetrics wrapper type for SVM

Cargo.toml

@@ -119,6 +119,7 @@ members = [
     "streamer",
     "svm",
     "svm-conformance",
+    "svm-rent-collector",
     "svm-transaction",
     "svm/examples/paytube",
     "test-validator",
@@ -434,6 +435,7 @@ solana-streamer = { path = "streamer", version = "=2.1.0" }
 solana-svm = { path = "svm", version = "=2.1.0" }
 solana-svm-conformance = { path = "svm-conformance", version = "=2.1.0" }
 solana-svm-example-paytube = { path = "svm/examples/paytube", version = "=2.1.0" }
+solana-svm-rent-collector = { path = "svm-rent-collector", version = "=2.1.0" }
 solana-svm-transaction = { path = "svm-transaction", version = "=2.1.0" }
 solana-system-program = { path = "programs/system", version = "=2.1.0" }
 solana-test-validator = { path = "test-validator", version = "=2.1.0" }

accounts-db/src/accounts_db.rs

@@ -1959,6 +1959,7 @@ struct CleanAccountsStats {
     remove_dead_accounts_shrink_us: AtomicU64,
     clean_stored_dead_slots_us: AtomicU64,
     uncleaned_roots_slot_list_1: AtomicU64,
+    get_account_sizes_us: AtomicU64,
 }
 
 impl CleanAccountsStats {
@@ -2030,6 +2031,7 @@ pub struct ShrinkStats {
     alive_accounts: AtomicU64,
     accounts_loaded: AtomicU64,
     purged_zero_lamports: AtomicU64,
+    accounts_not_found_in_index: AtomicU64,
 }
 
 impl ShrinkStats {
@@ -2133,6 +2135,11 @@ impl ShrinkStats {
                     self.purged_zero_lamports.swap(0, Ordering::Relaxed),
                     i64
                 ),
+                (
+                    "accounts_not_found_in_index",
+                    self.accounts_not_found_in_index.swap(0, Ordering::Relaxed),
+                    i64
+                ),
             );
         }
     }
@@ -2340,6 +2347,13 @@ impl ShrinkAncientStats {
                     .swap(0, Ordering::Relaxed),
                 i64
             ),
+            (
+                "accounts_not_found_in_index",
+                self.shrink_stats
+                    .accounts_not_found_in_index
+                    .swap(0, Ordering::Relaxed),
+                i64
+            ),
         );
     }
 }
@@ -3580,6 +3594,13 @@ impl AccountsDb {
                     .swap(0, Ordering::Relaxed),
                 i64
             ),
+            (
+                "get_account_sizes_us",
+                self.clean_accounts_stats
+                    .get_account_sizes_us
+                    .swap(0, Ordering::Relaxed),
+                i64
+            ),
             (
                 "clean_old_root_us",
                 self.clean_accounts_stats
@@ -3928,6 +3949,10 @@ impl AccountsDb {
                         alive_accounts.add(ref_count, stored_account, slot_list);
                         alive += 1;
                     }
+                } else {
+                    stats
+                        .accounts_not_found_in_index
+                        .fetch_add(1, Ordering::Relaxed);
                 }
                 index += 1;
                 result
@@ -3979,6 +4004,11 @@ impl AccountsDb {
         }
     }
 
+    #[cfg(feature = "dev-context-only-utils")]
+    pub fn set_storage_access(&mut self, storage_access: StorageAccess) {
+        self.storage_access = storage_access;
+    }
+
     /// Sort `accounts` by pubkey and removes all but the *last* of consecutive
     /// accounts in the vector with the same pubkey.
     ///
@@ -8021,20 +8051,25 @@ impl AccountsDb {
                     dead_slots.insert(*slot);
                 }
                 else {
-                    let mut offsets = offsets.iter().cloned().collect::<Vec<_>>();
-                    // sort so offsets are in order. This improves efficiency of loading the accounts.
-                    offsets.sort_unstable();
-                    let dead_bytes = store.accounts.get_account_sizes(&offsets).iter().sum();
-                    store.remove_accounts(dead_bytes, reset_accounts, offsets.len());
-                    if Self::is_shrinking_productive(*slot, &store)
-                        && self.is_candidate_for_shrink(&store)
-                    {
-                        // Checking that this single storage entry is ready for shrinking,
-                        // should be a sufficient indication that the slot is ready to be shrunk
-                        // because slots should only have one storage entry, namely the one that was
-                        // created by `flush_slot_cache()`.
-                        new_shrink_candidates.insert(*slot);
-                    }
+                    let (_, us) = measure_us!({
+                        let mut offsets = offsets.iter().cloned().collect::<Vec<_>>();
+                        // sort so offsets are in order. This improves efficiency of loading the accounts.
+                        offsets.sort_unstable();
+                        let dead_bytes = store.accounts.get_account_sizes(&offsets).iter().sum();
+                        store.remove_accounts(dead_bytes, reset_accounts, offsets.len());
+                        if Self::is_shrinking_productive(*slot, &store)
+                            && self.is_candidate_for_shrink(&store)
+                        {
+                            // Checking that this single storage entry is ready for shrinking,
+                            // should be a sufficient indication that the slot is ready to be shrunk
+                            // because slots should only have one storage entry, namely the one that was
+                            // created by `flush_slot_cache()`.
+                            new_shrink_candidates.insert(*slot);
+                        }
+                    });
+                    self.clean_accounts_stats
+                        .get_account_sizes_us
+                        .fetch_add(us, Ordering::Relaxed);
                 }
             }
         });

accounts-db/src/accounts_index/in_mem_accounts_index.rs

@@ -1255,14 +1255,24 @@ impl<T: IndexValue, U: DiskIndexValue + From<T> + Into<T>> InMemAccountsIndex<T,
                                     let slot_list = slot_list
                                         .take()
                                         .unwrap_or_else(|| v.slot_list.read().unwrap());
+                                    // Check the ref count and slot list one more time before flushing.
+                                    // It is possible the foreground has updated this entry since
+                                    // we last checked above in `should_evict_from_mem()`.
+                                    // If the entry *was* updated, re-mark it as dirty then
+                                    // skip to the next pubkey/entry.
+                                    let ref_count = v.ref_count();
+                                    if ref_count != 1 || slot_list.len() != 1 {
+                                        v.set_dirty(true);
+                                        break;
+                                    }
                                     disk.try_write(
                                         &k,
                                         (
                                             &slot_list
                                                 .iter()
                                                 .map(|(slot, info)| (*slot, (*info).into()))
                                                 .collect::<Vec<_>>(),
-                                            v.ref_count(),
+                                            ref_count,
                                         ),
                                     )
                                 };