accounts-db: Benchmark cache evictions

The already existing `concurrent_{read,scan}_write` benchmarks are not
sufficient for benchmarking the eviction and evaluating what kind of
eviction policy performs the best, because they don't fill up the cache,
so eviction never happens.

Add a new benchmark, which starts measuring the concurrent reads and
writes on a full cache.

Cargo.toml

@@ -367,6 +367,7 @@ merlin = "3"
 min-max-heap = "1.3.0"
 mockall = "0.11.4"
 modular-bitfield = "0.11.2"
+ndarray = "0.16.1"
 nix = "0.29.0"
 num-bigint = "0.4.6"
 num-derive = "0.4"

accounts-db/Cargo.toml

@@ -67,6 +67,7 @@ assert_matches = { workspace = true }
 criterion = { workspace = true }
 libsecp256k1 = { workspace = true }
 memoffset = { workspace = true }
+ndarray = { workspace = true }
 rand_chacha = { workspace = true }
 serde_bytes = { workspace = true }
 # See order-crates-for-publishing.py for using this unusual `path = "."`
@@ -103,6 +104,10 @@ harness = false
 name = "bench_hashing"
 harness = false
 
+[[bench]]
+name = "read_only_accounts_cache"
+harness = false
+
 [[bench]]
 name = "bench_serde"
 harness = false

accounts-db/benches/read_only_accounts_cache.rs

@@ -0,0 +1,184 @@
+#![feature(test)]
+
+extern crate test;
+
+use {
+    criterion::{criterion_group, criterion_main, BenchmarkId, Criterion},
+    rand::{rngs::SmallRng, seq::SliceRandom, SeedableRng},
+    solana_accounts_db::{
+        accounts_db::AccountsDb,
+        read_only_accounts_cache::{ReadOnlyAccountsCache, CACHE_ENTRY_SIZE},
+    },
+    solana_sdk::{
+        account::{Account, ReadableAccount},
+        pubkey,
+    },
+    std::{
+        iter,
+        sync::{
+            atomic::{AtomicBool, Ordering},
+            Arc,
+        },
+        thread::Builder,
+    },
+};
+
+/// Sizes of accounts to bench.
+const ACCOUNTS_SIZES: &[usize] = &[0, 512, 1024];
+/// Numbers of reader and writer threads to bench.
+const NUM_READERS_WRITERS: &[usize] = &[
+    8,
+    16,
+    // These parameters are likely to freeze your computer, if it has less than
+    // 32 cores.
+    // 32, 64, 128, 256, 512, 1024,
+];
+
+/// Benchmarks the read-only cache eviction mechanism. It does so by performing
+/// multithreaded reads and writes on a full cache. Each write triggers
+/// eviction. Background reads add more contention.
+fn bench_read_only_accounts_cache_eviction(c: &mut Criterion) {
+    /// Number of accounts to use in the benchmark. That's the maximum number
+    /// of evictions observed on mainnet validators.
+    const NUM_ACCOUNTS_BENCHED: usize = 1000;
+
+    let mut group = c.benchmark_group("cache_eviction");
+
+    for account_size in ACCOUNTS_SIZES {
+        // Number of accounts needed to initially fill the cache.
+        let num_accounts_init = AccountsDb::DEFAULT_MAX_READ_ONLY_CACHE_DATA_SIZE_HI
+            .div_ceil(CACHE_ENTRY_SIZE.saturating_add(*account_size));
+
+        for num_readers_writers in NUM_READERS_WRITERS {
+            let cache = Arc::new(ReadOnlyAccountsCache::new(
+                AccountsDb::DEFAULT_MAX_READ_ONLY_CACHE_DATA_SIZE_LO,
+                AccountsDb::DEFAULT_MAX_READ_ONLY_CACHE_DATA_SIZE_HI,
+                AccountsDb::READ_ONLY_CACHE_MS_TO_SKIP_LRU_UPDATE,
+            ));
+
+            // Prepare accounts for the cache fillup.
+            let pubkeys: Vec<_> = iter::repeat_with(pubkey::new_rand)
+                .take(num_accounts_init)
+                .collect();
+            let accounts_data = iter::repeat(
+                Account {
+                    lamports: 1,
+                    data: vec![1; *account_size],
+                    ..Default::default()
+                }
+                .into(),
+            )
+            .take(num_accounts_init);
+            let accounts = pubkeys.iter().zip(accounts_data);
+
+            // Fill up the cache.
+            let slot = 0;
+            for (pubkey, account) in accounts {
+                cache.store(*pubkey, slot, account);
+            }
+
+            // Spawn the reader and writer threads in the background. They are
+            // going to read and write
+            let stop_threads = Arc::new(AtomicBool::new(false));
+            let reader_handles = (0..*num_readers_writers).map(|i| {
+                let stop_threads = Arc::clone(&stop_threads);
+                let cache = Arc::clone(&cache);
+                let pubkeys = pubkeys.clone();
+
+                Builder::new()
+                    .name(format!("reader{i:02}"))
+                    .spawn({
+                        move || {
+                            // Continuously read random accounts.
+                            let mut rng = SmallRng::seed_from_u64(i as u64);
+                            while !stop_threads.load(Ordering::Relaxed) {
+                                let pubkey = pubkeys.choose(&mut rng).unwrap();
+                                test::black_box(cache.load(*pubkey, slot));
+                            }
+                        }
+                    })
+                    .unwrap()
+            });
+            let slot = 1;
+            let writer_handles = (0..*num_readers_writers).map(|i| {
+                let stop_threads = Arc::clone(&stop_threads);
+                let cache = Arc::clone(&cache);
+                let pubkeys = pubkeys.clone();
+
+                Builder::new()
+                    .name(format!("writer{i:02}"))
+                    .spawn({
+                        move || {
+                            // Continuously write to already existing pubkeys.
+                            let mut rng = SmallRng::seed_from_u64(100_u64.saturating_add(i as u64));
+                            while !stop_threads.load(Ordering::Relaxed) {
+                                let pubkey = pubkeys.choose(&mut rng).unwrap();
+                                cache.store(
+                                    *pubkey,
+                                    slot,
+                                    Account {
+                                        lamports: 1,
+                                        data: vec![1; *account_size],
+                                        ..Default::default()
+                                    }
+                                    .to_account_shared_data(),
+                                );
+                            }
+                        }
+                    })
+                    .unwrap()
+            });
+
+            // Benchmark loads and stores on a full cache, trigerring eviction
+            // on each write. The background threads add contention.
+            let accounts: Vec<_> = iter::repeat((
+                pubkey::new_rand(),
+                Account {
+                    lamports: 1,
+                    data: vec![1; *account_size],
+                    ..Default::default()
+                }
+                .to_account_shared_data(),
+            ))
+            .take(NUM_ACCOUNTS_BENCHED)
+            .collect();
+            group.bench_function(
+                BenchmarkId::new(
+                    "read_only_accounts_cache_eviction_store",
+                    format!("{account_size}_bytes_{num_readers_writers}_threads"),
+                ),
+                |b| {
+                    b.iter(|| {
+                        for (pubkey, account) in accounts.iter() {
+                            cache.store(pubkey.to_owned(), slot, account.to_owned());
+                        }
+                    })
+                },
+            );
+            group.bench_function(
+                BenchmarkId::new(
+                    "read_only_accounts_cache_eviction_load",
+                    format!("{account_size}_bytes_{num_readers_writers}_threads"),
+                ),
+                |b| {
+                    b.iter(|| {
+                        for (pubkey, _) in accounts.iter() {
+                            test::black_box(cache.load(pubkey.to_owned(), slot));
+                        }
+                    })
+                },
+            );
+
+            stop_threads.store(true, Ordering::Relaxed);
+            for reader_handle in reader_handles {
+                reader_handle.join().unwrap();
+            }
+            for writer_handle in writer_handles {
+                writer_handle.join().unwrap();
+            }
+        }
+    }
+}
+
+criterion_group!(benches, bench_read_only_accounts_cache_eviction);
+criterion_main!(benches);

accounts-db/src/accounts_db.rs

@@ -1889,11 +1889,14 @@ impl AccountsDb {
     pub const DEFAULT_ACCOUNTS_HASH_CACHE_DIR: &'static str = "accounts_hash_cache";
 
     // read only cache does not update lru on read of an entry unless it has been at least this many ms since the last lru update
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     const READ_ONLY_CACHE_MS_TO_SKIP_LRU_UPDATE: u32 = 100;
 
     // The default high and low watermark sizes for the accounts read cache.
     // If the cache size exceeds MAX_SIZE_HI, it'll evict entries until the size is <= MAX_SIZE_LO.
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     const DEFAULT_MAX_READ_ONLY_CACHE_DATA_SIZE_LO: usize = 400 * 1024 * 1024;
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     const DEFAULT_MAX_READ_ONLY_CACHE_DATA_SIZE_HI: usize = 410 * 1024 * 1024;
 
     pub fn default_for_tests() -> Self {

accounts-db/src/lib.rs

@@ -32,6 +32,9 @@ mod file_io;
 pub mod hardened_unpack;
 pub mod partitioned_rewards;
 pub mod pubkey_bins;
+#[cfg(feature = "dev-context-only-utils")]
+pub mod read_only_accounts_cache;
+#[cfg(not(feature = "dev-context-only-utils"))]
 mod read_only_accounts_cache;
 mod rolling_bit_field;
 pub mod secondary_index;

accounts-db/src/read_only_accounts_cache.rs

@@ -1,5 +1,7 @@
 //! ReadOnlyAccountsCache used to store accounts, such as executable accounts,
 //! which can be large, loaded many times, and rarely change.
+#[cfg(feature = "dev-context-only-utils")]
+use qualifier_attr::qualifiers;
 use {
     dashmap::{mapref::entry::Entry, DashMap},
     index_list::{Index, IndexList},
@@ -22,6 +24,7 @@ use {
     },
 };
 
+#[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
 const CACHE_ENTRY_SIZE: usize =
     std::mem::size_of::<ReadOnlyAccountCacheEntry>() + 2 * std::mem::size_of::<ReadOnlyCacheKey>();
 
@@ -65,6 +68,7 @@ struct AtomicReadOnlyCacheStats {
     evictor_wakeup_count_productive: AtomicU64,
 }
 
+#[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
 #[derive(Debug)]
 pub(crate) struct ReadOnlyAccountsCache {
     cache: Arc<DashMap<ReadOnlyCacheKey, ReadOnlyAccountCacheEntry>>,
@@ -93,6 +97,7 @@ pub(crate) struct ReadOnlyAccountsCache {
 }
 
 impl ReadOnlyAccountsCache {
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     pub(crate) fn new(
         max_data_size_lo: usize,
         max_data_size_hi: usize,
@@ -137,6 +142,7 @@ impl ReadOnlyAccountsCache {
         }
     }
 
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     pub(crate) fn load(&self, pubkey: Pubkey, slot: Slot) -> Option<AccountSharedData> {
         let (account, load_us) = measure_us!({
             let mut found = None;
@@ -175,6 +181,7 @@ impl ReadOnlyAccountsCache {
         CACHE_ENTRY_SIZE + account.data().len()
     }
 
+    #[cfg_attr(feature = "dev-context-only-utils", qualifiers(pub))]
     pub(crate) fn store(&self, pubkey: Pubkey, slot: Slot, account: AccountSharedData) {
         let measure_store = Measure::start("");
         self.highest_slot_stored.fetch_max(slot, Ordering::Release);