From c4f03ee5b158dc351aa7fab8af435e5236e4c877 Mon Sep 17 00:00:00 2001 From: steviez Date: Wed, 23 Oct 2024 02:31:55 -0500 Subject: [PATCH 1/6] Add arg for clean --- accounts-db/src/accounts_db.rs | 16 +++++++++++++++- validator/src/cli/thread_args.rs | 20 ++++++++++++++++++++ validator/src/main.rs | 2 ++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 29236a998455ba..ad4b6a0c694edb 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -510,6 +510,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig { storage_access: StorageAccess::Mmap, scan_filter_for_shrinking: ScanFilter::OnlyAbnormalWithVerify, enable_experimental_accumulator_hash: false, + num_clean_threads: None, num_hash_threads: None, }; pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig { @@ -528,6 +529,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig storage_access: StorageAccess::Mmap, scan_filter_for_shrinking: ScanFilter::OnlyAbnormalWithVerify, enable_experimental_accumulator_hash: false, + num_clean_threads: None, num_hash_threads: None, }; @@ -640,6 +642,8 @@ pub struct AccountsDbConfig { pub storage_access: StorageAccess, pub scan_filter_for_shrinking: ScanFilter, pub enable_experimental_accumulator_hash: bool, + /// Number of threads for background cleaning operations (`thread_pool_clean') + pub num_clean_threads: Option, /// Number of threads for background accounts hashing (`thread_pool_hash`) pub num_hash_threads: Option, } @@ -1902,7 +1906,17 @@ impl AccountsDb { .stack_size(ACCOUNTS_STACK_SIZE) .build() .expect("new rayon threadpool"); - let thread_pool_clean = make_min_priority_thread_pool(); + + let num_clean_threads = accounts_db_config + .num_clean_threads + .map(Into::into) + .unwrap_or_else(quarter_thread_count); + let thread_pool_clean = rayon::ThreadPoolBuilder::new() + .thread_name(|i| format!("solAccountsLo{i:02}")) + .num_threads(num_clean_threads) + .build() + .expect("new rayon threadpool"); + let thread_pool_hash = make_hash_thread_pool(accounts_db_config.num_hash_threads); let mut new = Self { diff --git a/validator/src/cli/thread_args.rs b/validator/src/cli/thread_args.rs index 21de36f4abe8d6..51545e8786fb75 100644 --- a/validator/src/cli/thread_args.rs +++ b/validator/src/cli/thread_args.rs @@ -10,6 +10,7 @@ use { // Need this struct to provide &str whose lifetime matches that of the CLAP Arg's pub struct DefaultThreadArgs { + pub accounts_db_clean_threads: String, pub accounts_db_hash_threads: String, pub ip_echo_server_threads: String, pub replay_forks_threads: String, @@ -21,6 +22,7 @@ pub struct DefaultThreadArgs { impl Default for DefaultThreadArgs { fn default() -> Self { Self { + accounts_db_clean_threads: AccountsDbCleanThreadsArg::bounded_default().to_string(), accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), ip_echo_server_threads: IpEchoServerThreadsArg::bounded_default().to_string(), replay_forks_threads: ReplayForksThreadsArg::bounded_default().to_string(), @@ -34,6 +36,7 @@ impl Default for DefaultThreadArgs { pub fn thread_args<'a>(defaults: &DefaultThreadArgs) -> Vec> { vec![ + new_thread_arg::(&defaults.accounts_db_clean_threads), new_thread_arg::(&defaults.accounts_db_hash_threads), new_thread_arg::(&defaults.ip_echo_server_threads), new_thread_arg::(&defaults.replay_forks_threads), @@ -55,6 +58,7 @@ fn new_thread_arg<'a, T: ThreadArg>(default: &str) -> Arg<'_, 'a> { } pub struct NumThreadConfig { + pub accounts_db_clean_threads: NonZeroUsize, pub accounts_db_hash_threads: NonZeroUsize, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, @@ -65,6 +69,11 @@ pub struct NumThreadConfig { pub fn parse_num_threads_args(matches: &ArgMatches) -> NumThreadConfig { NumThreadConfig { + accounts_db_clean_threads: value_t_or_exit!( + matches, + AccountsDbCleanThreadsArg::NAME, + NonZeroUsize + ), accounts_db_hash_threads: value_t_or_exit!( matches, AccountsDbHashThreadsArg::NAME, @@ -126,6 +135,17 @@ trait ThreadArg { } } +struct AccountsDbCleanThreadsArg; +impl ThreadArg for AccountsDbCleanThreadsArg { + const NAME: &'static str = "accounts_db_clean_threads"; + const LONG_NAME: &'static str = "accounts-db-clean-threads"; + const HELP: &'static str = "Number of threads to use for cleaning AccountsDb"; + + fn default() -> usize { + accounts_db::quarter_thread_count() + } +} + struct AccountsDbHashThreadsArg; impl ThreadArg for AccountsDbHashThreadsArg { const NAME: &'static str = "accounts_db_hash_threads"; diff --git a/validator/src/main.rs b/validator/src/main.rs index 6d185f8c8e9bb3..f7de2dd09f18e7 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -910,6 +910,7 @@ pub fn main() { }; let cli::thread_args::NumThreadConfig { + accounts_db_clean_threads, accounts_db_hash_threads, ip_echo_server_threads, replay_forks_threads, @@ -1312,6 +1313,7 @@ pub fn main() { scan_filter_for_shrinking, enable_experimental_accumulator_hash: matches .is_present("accounts_db_experimental_accumulator_hash"), + num_clean_threads: Some(accounts_db_clean_threads), num_hash_threads: Some(accounts_db_hash_threads), ..AccountsDbConfig::default() }; From 7a4d450b81ba2a615004c42420c8786497104bcd Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 24 Oct 2024 13:32:29 -0500 Subject: [PATCH 2/6] Add arg for foreground pool --- accounts-db/src/accounts_db.rs | 14 +++++++++++++- validator/src/cli/thread_args.rs | 20 ++++++++++++++++++++ validator/src/main.rs | 2 ++ 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index ad4b6a0c694edb..f679d8fc266efe 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -512,6 +512,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig { enable_experimental_accumulator_hash: false, num_clean_threads: None, num_hash_threads: None, + num_process_threads: None, }; pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig { index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS), @@ -531,6 +532,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig enable_experimental_accumulator_hash: false, num_clean_threads: None, num_hash_threads: None, + num_process_threads: None, }; pub type BinnedHashData = Vec>; @@ -646,6 +648,8 @@ pub struct AccountsDbConfig { pub num_clean_threads: Option, /// Number of threads for background accounts hashing (`thread_pool_hash`) pub num_hash_threads: Option, + /// Number of threads for foreground opeations (`thread_pool`) + pub num_process_threads: Option, } #[cfg(not(test))] @@ -1770,6 +1774,10 @@ pub fn make_hash_thread_pool(num_threads: Option) -> ThreadPool { .unwrap() } +pub fn default_num_foreground_threads() -> usize { + get_thread_count() +} + #[cfg(feature = "frozen-abi")] impl solana_frozen_abi::abi_example::AbiExample for AccountsDb { fn example() -> Self { @@ -1900,8 +1908,12 @@ impl AccountsDb { // Increase the stack for accounts threads // rayon needs a lot of stack const ACCOUNTS_STACK_SIZE: usize = 8 * 1024 * 1024; + let num_process_threads = accounts_db_config + .num_process_threads + .map(Into::into) + .unwrap_or_else(default_num_foreground_threads); let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(get_thread_count()) + .num_threads(num_process_threads) .thread_name(|i| format!("solAccounts{i:02}")) .stack_size(ACCOUNTS_STACK_SIZE) .build() diff --git a/validator/src/cli/thread_args.rs b/validator/src/cli/thread_args.rs index 51545e8786fb75..c85247f40d615e 100644 --- a/validator/src/cli/thread_args.rs +++ b/validator/src/cli/thread_args.rs @@ -12,6 +12,7 @@ use { pub struct DefaultThreadArgs { pub accounts_db_clean_threads: String, pub accounts_db_hash_threads: String, + pub accounts_db_process_threads: String, pub ip_echo_server_threads: String, pub replay_forks_threads: String, pub replay_transactions_threads: String, @@ -24,6 +25,7 @@ impl Default for DefaultThreadArgs { Self { accounts_db_clean_threads: AccountsDbCleanThreadsArg::bounded_default().to_string(), accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), + accounts_db_process_threads: AccountsDbProcessThreadsArg::bounded_default().to_string(), ip_echo_server_threads: IpEchoServerThreadsArg::bounded_default().to_string(), replay_forks_threads: ReplayForksThreadsArg::bounded_default().to_string(), replay_transactions_threads: ReplayTransactionsThreadsArg::bounded_default() @@ -38,6 +40,7 @@ pub fn thread_args<'a>(defaults: &DefaultThreadArgs) -> Vec> { vec![ new_thread_arg::(&defaults.accounts_db_clean_threads), new_thread_arg::(&defaults.accounts_db_hash_threads), + new_thread_arg::(&defaults.accounts_db_process_threads), new_thread_arg::(&defaults.ip_echo_server_threads), new_thread_arg::(&defaults.replay_forks_threads), new_thread_arg::(&defaults.replay_transactions_threads), @@ -60,6 +63,7 @@ fn new_thread_arg<'a, T: ThreadArg>(default: &str) -> Arg<'_, 'a> { pub struct NumThreadConfig { pub accounts_db_clean_threads: NonZeroUsize, pub accounts_db_hash_threads: NonZeroUsize, + pub accounts_db_process_threads: NonZeroUsize, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, pub replay_transactions_threads: NonZeroUsize, @@ -79,6 +83,11 @@ pub fn parse_num_threads_args(matches: &ArgMatches) -> NumThreadConfig { AccountsDbHashThreadsArg::NAME, NonZeroUsize ), + accounts_db_process_threads: value_t_or_exit!( + matches, + AccountsDbProcessThreadsArg::NAME, + NonZeroUsize + ), ip_echo_server_threads: value_t_or_exit!( matches, IpEchoServerThreadsArg::NAME, @@ -157,6 +166,17 @@ impl ThreadArg for AccountsDbHashThreadsArg { } } +struct AccountsDbProcessThreadsArg; +impl ThreadArg for AccountsDbProcessThreadsArg { + const NAME: &'static str = "accounts_db_process_threads"; + const LONG_NAME: &'static str = "accounts-db-process-threads"; + const HELP: &'static str = "Number of threads to use for AccountsDb block processing"; + + fn default() -> usize { + accounts_db::default_num_foreground_threads() + } +} + struct IpEchoServerThreadsArg; impl ThreadArg for IpEchoServerThreadsArg { const NAME: &'static str = "ip_echo_server_threads"; diff --git a/validator/src/main.rs b/validator/src/main.rs index f7de2dd09f18e7..6108ba716d77e6 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -912,6 +912,7 @@ pub fn main() { let cli::thread_args::NumThreadConfig { accounts_db_clean_threads, accounts_db_hash_threads, + accounts_db_process_threads, ip_echo_server_threads, replay_forks_threads, replay_transactions_threads, @@ -1315,6 +1316,7 @@ pub fn main() { .is_present("accounts_db_experimental_accumulator_hash"), num_clean_threads: Some(accounts_db_clean_threads), num_hash_threads: Some(accounts_db_hash_threads), + num_process_threads: Some(accounts_db_process_threads), ..AccountsDbConfig::default() }; From bc2655f1cbf70b9b68706712a9cb94b444e87073 Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 24 Oct 2024 14:39:07 -0500 Subject: [PATCH 3/6] Add arg for accounts index flush threads --- accounts-db/src/accounts_index.rs | 14 ++++++++++---- accounts-db/src/accounts_index_storage.rs | 23 ++++++++++------------- validator/src/cli/thread_args.rs | 23 ++++++++++++++++++++++- validator/src/main.rs | 2 ++ 4 files changed, 44 insertions(+), 18 deletions(-) diff --git a/accounts-db/src/accounts_index.rs b/accounts-db/src/accounts_index.rs index a849d08af385f6..ff430931c9ec48 100644 --- a/accounts-db/src/accounts_index.rs +++ b/accounts-db/src/accounts_index.rs @@ -27,6 +27,7 @@ use { std::{ collections::{btree_map::BTreeMap, HashSet}, fmt::Debug, + num::NonZeroUsize, ops::{ Bound, Bound::{Excluded, Included, Unbounded}, @@ -45,10 +46,11 @@ pub const ITER_BATCH_SIZE: usize = 1000; pub const BINS_DEFAULT: usize = 8192; pub const BINS_FOR_TESTING: usize = 2; // we want > 1, but each bin is a few disk files with a disk based index, so fewer is better pub const BINS_FOR_BENCHMARKS: usize = 8192; -pub const FLUSH_THREADS_TESTING: usize = 1; +// The unsafe is safe because we're using a fixed, known non-zero value +pub const FLUSH_THREADS_TESTING: NonZeroUsize = unsafe { NonZeroUsize::new_unchecked(1) }; pub const ACCOUNTS_INDEX_CONFIG_FOR_TESTING: AccountsIndexConfig = AccountsIndexConfig { bins: Some(BINS_FOR_TESTING), - flush_threads: Some(FLUSH_THREADS_TESTING), + num_flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, index_limit_mb: IndexLimitMb::Unlimited, ages_to_stay_in_cache: None, @@ -57,7 +59,7 @@ pub const ACCOUNTS_INDEX_CONFIG_FOR_TESTING: AccountsIndexConfig = AccountsIndex }; pub const ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS: AccountsIndexConfig = AccountsIndexConfig { bins: Some(BINS_FOR_BENCHMARKS), - flush_threads: Some(FLUSH_THREADS_TESTING), + num_flush_threads: Some(FLUSH_THREADS_TESTING), drives: None, index_limit_mb: IndexLimitMb::Unlimited, ages_to_stay_in_cache: None, @@ -218,7 +220,7 @@ pub enum IndexLimitMb { #[derive(Debug, Default, Clone)] pub struct AccountsIndexConfig { pub bins: Option, - pub flush_threads: Option, + pub num_flush_threads: Option, pub drives: Option>, pub index_limit_mb: IndexLimitMb, pub ages_to_stay_in_cache: Option, @@ -227,6 +229,10 @@ pub struct AccountsIndexConfig { pub started_from_validator: bool, } +pub fn default_num_flush_threads() -> NonZeroUsize { + NonZeroUsize::new(std::cmp::max(2, num_cpus::get() / 4)).expect("non-zero system threads") +} + #[derive(Debug, Default, Clone)] pub struct AccountSecondaryIndexes { pub keys: Option, diff --git a/accounts-db/src/accounts_index_storage.rs b/accounts-db/src/accounts_index_storage.rs index 3a654c84c25f97..48c5f55c5d9f64 100644 --- a/accounts-db/src/accounts_index_storage.rs +++ b/accounts-db/src/accounts_index_storage.rs @@ -1,7 +1,7 @@ use { crate::{ accounts_index::{ - in_mem_accounts_index::InMemAccountsIndex, AccountsIndexConfig, DiskIndexValue, + self, in_mem_accounts_index::InMemAccountsIndex, AccountsIndexConfig, DiskIndexValue, IndexValue, }, bucket_map_holder::BucketMapHolder, @@ -9,6 +9,7 @@ use { }, std::{ fmt::Debug, + num::NonZeroUsize, sync::{ atomic::{AtomicBool, Ordering}, Arc, Mutex, @@ -58,14 +59,14 @@ impl BgThreads { fn new + Into>( storage: &Arc>, in_mem: &[Arc>], - threads: usize, + threads: NonZeroUsize, can_advance_age: bool, exit: Arc, ) -> Self { // stop signal used for THIS batch of bg threads let local_exit = Arc::new(AtomicBool::default()); let handles = Some( - (0..threads) + (0..threads.get()) .map(|idx| { // the first thread we start is special let can_advance_age = can_advance_age && idx == 0; @@ -123,7 +124,7 @@ impl + Into> AccountsIndexStorage< *self.startup_worker_threads.lock().unwrap() = Some(BgThreads::new( &self.storage, &self.in_mem, - Self::num_threads(), + accounts_index::default_num_flush_threads(), false, // cannot advance age from any of these threads self.exit.clone(), )); @@ -151,25 +152,21 @@ impl + Into> AccountsIndexStorage< self.in_mem.iter().for_each(|mem| mem.shrink_to_fit()) } - fn num_threads() -> usize { - std::cmp::max(2, num_cpus::get() / 4) - } - /// allocate BucketMapHolder and InMemAccountsIndex[] pub fn new(bins: usize, config: &Option, exit: Arc) -> Self { - let threads = config + let num_flush_threads = config .as_ref() - .and_then(|config| config.flush_threads) - .unwrap_or_else(Self::num_threads); + .and_then(|config| config.num_flush_threads) + .unwrap_or_else(accounts_index::default_num_flush_threads); - let storage = Arc::new(BucketMapHolder::new(bins, config, threads)); + let storage = Arc::new(BucketMapHolder::new(bins, config, num_flush_threads.get())); let in_mem = (0..bins) .map(|bin| Arc::new(InMemAccountsIndex::new(&storage, bin))) .collect::>(); Self { - _bg_threads: BgThreads::new(&storage, &in_mem, threads, true, exit.clone()), + _bg_threads: BgThreads::new(&storage, &in_mem, num_flush_threads, true, exit.clone()), storage, in_mem, startup_worker_threads: Mutex::default(), diff --git a/validator/src/cli/thread_args.rs b/validator/src/cli/thread_args.rs index c85247f40d615e..970ec9579a58c1 100644 --- a/validator/src/cli/thread_args.rs +++ b/validator/src/cli/thread_args.rs @@ -2,7 +2,7 @@ use { clap::{value_t_or_exit, Arg, ArgMatches}, - solana_accounts_db::accounts_db, + solana_accounts_db::{accounts_db, accounts_index}, solana_clap_utils::{hidden_unless_forced, input_validators::is_within_range}, solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, std::{num::NonZeroUsize, ops::RangeInclusive}, @@ -13,6 +13,7 @@ pub struct DefaultThreadArgs { pub accounts_db_clean_threads: String, pub accounts_db_hash_threads: String, pub accounts_db_process_threads: String, + pub accounts_index_flush_threads: String, pub ip_echo_server_threads: String, pub replay_forks_threads: String, pub replay_transactions_threads: String, @@ -26,6 +27,8 @@ impl Default for DefaultThreadArgs { accounts_db_clean_threads: AccountsDbCleanThreadsArg::bounded_default().to_string(), accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), accounts_db_process_threads: AccountsDbProcessThreadsArg::bounded_default().to_string(), + accounts_index_flush_threads: AccountsIndexFlushThreadsArg::bounded_default() + .to_string(), ip_echo_server_threads: IpEchoServerThreadsArg::bounded_default().to_string(), replay_forks_threads: ReplayForksThreadsArg::bounded_default().to_string(), replay_transactions_threads: ReplayTransactionsThreadsArg::bounded_default() @@ -41,6 +44,7 @@ pub fn thread_args<'a>(defaults: &DefaultThreadArgs) -> Vec> { new_thread_arg::(&defaults.accounts_db_clean_threads), new_thread_arg::(&defaults.accounts_db_hash_threads), new_thread_arg::(&defaults.accounts_db_process_threads), + new_thread_arg::(&defaults.accounts_db_process_threads), new_thread_arg::(&defaults.ip_echo_server_threads), new_thread_arg::(&defaults.replay_forks_threads), new_thread_arg::(&defaults.replay_transactions_threads), @@ -64,6 +68,7 @@ pub struct NumThreadConfig { pub accounts_db_clean_threads: NonZeroUsize, pub accounts_db_hash_threads: NonZeroUsize, pub accounts_db_process_threads: NonZeroUsize, + pub accounts_index_flush_threads: NonZeroUsize, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, pub replay_transactions_threads: NonZeroUsize, @@ -88,6 +93,11 @@ pub fn parse_num_threads_args(matches: &ArgMatches) -> NumThreadConfig { AccountsDbProcessThreadsArg::NAME, NonZeroUsize ), + accounts_index_flush_threads: value_t_or_exit!( + matches, + AccountsIndexFlushThreadsArg::NAME, + NonZeroUsize + ), ip_echo_server_threads: value_t_or_exit!( matches, IpEchoServerThreadsArg::NAME, @@ -177,6 +187,17 @@ impl ThreadArg for AccountsDbProcessThreadsArg { } } +struct AccountsIndexFlushThreadsArg; +impl ThreadArg for AccountsIndexFlushThreadsArg { + const NAME: &'static str = "accounts_index_flush_threads"; + const LONG_NAME: &'static str = "accounts-index-flush-threads"; + const HELP: &'static str = "Number of threads to use for flushing the accounts index"; + + fn default() -> usize { + accounts_index::default_num_flush_threads().get() + } +} + struct IpEchoServerThreadsArg; impl ThreadArg for IpEchoServerThreadsArg { const NAME: &'static str = "ip_echo_server_threads"; diff --git a/validator/src/main.rs b/validator/src/main.rs index 6108ba716d77e6..06cd91c19b4b84 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -913,6 +913,7 @@ pub fn main() { accounts_db_clean_threads, accounts_db_hash_threads, accounts_db_process_threads, + accounts_index_flush_threads, ip_echo_server_threads, replay_forks_threads, replay_transactions_threads, @@ -1183,6 +1184,7 @@ pub fn main() { let mut accounts_index_config = AccountsIndexConfig { started_from_validator: true, // this is the only place this is set + num_flush_threads: Some(accounts_index_flush_threads), ..AccountsIndexConfig::default() }; if let Ok(bins) = value_t!(matches, "accounts_index_bins", usize) { From ba1137c0a52ec26d2dabe075734594f9c0bb759d Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 24 Oct 2024 15:35:36 -0500 Subject: [PATCH 4/6] pr feedback: typo fix --- accounts-db/src/accounts_db.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index f679d8fc266efe..ed95138fd2fc55 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -648,7 +648,7 @@ pub struct AccountsDbConfig { pub num_clean_threads: Option, /// Number of threads for background accounts hashing (`thread_pool_hash`) pub num_hash_threads: Option, - /// Number of threads for foreground opeations (`thread_pool`) + /// Number of threads for foreground operations (`thread_pool`) pub num_process_threads: Option, } From e491e013bcf83f74405b7557f02be42a9869772c Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 24 Oct 2024 16:24:30 -0500 Subject: [PATCH 5/6] process ==> foreground --- accounts-db/src/accounts_db.rs | 14 +++++++------- validator/src/cli/thread_args.rs | 23 ++++++++++++----------- validator/src/main.rs | 4 ++-- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index ed95138fd2fc55..56754ff9711275 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -512,7 +512,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig { enable_experimental_accumulator_hash: false, num_clean_threads: None, num_hash_threads: None, - num_process_threads: None, + num_foreground_threads: None, }; pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig { index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS), @@ -532,7 +532,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig enable_experimental_accumulator_hash: false, num_clean_threads: None, num_hash_threads: None, - num_process_threads: None, + num_foreground_threads: None, }; pub type BinnedHashData = Vec>; @@ -649,7 +649,7 @@ pub struct AccountsDbConfig { /// Number of threads for background accounts hashing (`thread_pool_hash`) pub num_hash_threads: Option, /// Number of threads for foreground operations (`thread_pool`) - pub num_process_threads: Option, + pub num_foreground_threads: Option, } #[cfg(not(test))] @@ -1905,15 +1905,15 @@ impl AccountsDb { let bank_hash_stats = Mutex::new(HashMap::from([(0, BankHashStats::default())])); - // Increase the stack for accounts threads + // Increase the stack for foreground threads // rayon needs a lot of stack const ACCOUNTS_STACK_SIZE: usize = 8 * 1024 * 1024; - let num_process_threads = accounts_db_config - .num_process_threads + let num_foreground_threads = accounts_db_config + .num_foreground_threads .map(Into::into) .unwrap_or_else(default_num_foreground_threads); let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(num_process_threads) + .num_threads(num_foreground_threads) .thread_name(|i| format!("solAccounts{i:02}")) .stack_size(ACCOUNTS_STACK_SIZE) .build() diff --git a/validator/src/cli/thread_args.rs b/validator/src/cli/thread_args.rs index 970ec9579a58c1..72cda6d6e71d4b 100644 --- a/validator/src/cli/thread_args.rs +++ b/validator/src/cli/thread_args.rs @@ -12,7 +12,7 @@ use { pub struct DefaultThreadArgs { pub accounts_db_clean_threads: String, pub accounts_db_hash_threads: String, - pub accounts_db_process_threads: String, + pub accounts_db_foreground_threads: String, pub accounts_index_flush_threads: String, pub ip_echo_server_threads: String, pub replay_forks_threads: String, @@ -26,7 +26,8 @@ impl Default for DefaultThreadArgs { Self { accounts_db_clean_threads: AccountsDbCleanThreadsArg::bounded_default().to_string(), accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), - accounts_db_process_threads: AccountsDbProcessThreadsArg::bounded_default().to_string(), + accounts_db_foreground_threads: AccountsDbForegroundThreadsArg::bounded_default() + .to_string(), accounts_index_flush_threads: AccountsIndexFlushThreadsArg::bounded_default() .to_string(), ip_echo_server_threads: IpEchoServerThreadsArg::bounded_default().to_string(), @@ -43,8 +44,8 @@ pub fn thread_args<'a>(defaults: &DefaultThreadArgs) -> Vec> { vec![ new_thread_arg::(&defaults.accounts_db_clean_threads), new_thread_arg::(&defaults.accounts_db_hash_threads), - new_thread_arg::(&defaults.accounts_db_process_threads), - new_thread_arg::(&defaults.accounts_db_process_threads), + new_thread_arg::(&defaults.accounts_db_foreground_threads), + new_thread_arg::(&defaults.accounts_db_foreground_threads), new_thread_arg::(&defaults.ip_echo_server_threads), new_thread_arg::(&defaults.replay_forks_threads), new_thread_arg::(&defaults.replay_transactions_threads), @@ -67,7 +68,7 @@ fn new_thread_arg<'a, T: ThreadArg>(default: &str) -> Arg<'_, 'a> { pub struct NumThreadConfig { pub accounts_db_clean_threads: NonZeroUsize, pub accounts_db_hash_threads: NonZeroUsize, - pub accounts_db_process_threads: NonZeroUsize, + pub accounts_db_foreground_threads: NonZeroUsize, pub accounts_index_flush_threads: NonZeroUsize, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, @@ -88,9 +89,9 @@ pub fn parse_num_threads_args(matches: &ArgMatches) -> NumThreadConfig { AccountsDbHashThreadsArg::NAME, NonZeroUsize ), - accounts_db_process_threads: value_t_or_exit!( + accounts_db_foreground_threads: value_t_or_exit!( matches, - AccountsDbProcessThreadsArg::NAME, + AccountsDbForegroundThreadsArg::NAME, NonZeroUsize ), accounts_index_flush_threads: value_t_or_exit!( @@ -176,10 +177,10 @@ impl ThreadArg for AccountsDbHashThreadsArg { } } -struct AccountsDbProcessThreadsArg; -impl ThreadArg for AccountsDbProcessThreadsArg { - const NAME: &'static str = "accounts_db_process_threads"; - const LONG_NAME: &'static str = "accounts-db-process-threads"; +struct AccountsDbForegroundThreadsArg; +impl ThreadArg for AccountsDbForegroundThreadsArg { + const NAME: &'static str = "accounts_db_foreground_threads"; + const LONG_NAME: &'static str = "accounts-db-foreground-threads"; const HELP: &'static str = "Number of threads to use for AccountsDb block processing"; fn default() -> usize { diff --git a/validator/src/main.rs b/validator/src/main.rs index 06cd91c19b4b84..ca152f4b09d71f 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -912,7 +912,7 @@ pub fn main() { let cli::thread_args::NumThreadConfig { accounts_db_clean_threads, accounts_db_hash_threads, - accounts_db_process_threads, + accounts_db_foreground_threads, accounts_index_flush_threads, ip_echo_server_threads, replay_forks_threads, @@ -1318,7 +1318,7 @@ pub fn main() { .is_present("accounts_db_experimental_accumulator_hash"), num_clean_threads: Some(accounts_db_clean_threads), num_hash_threads: Some(accounts_db_hash_threads), - num_process_threads: Some(accounts_db_process_threads), + num_foreground_threads: Some(accounts_db_foreground_threads), ..AccountsDbConfig::default() }; From 45b3b378b19b625f59639b4c1e8c55ea5271c3bf Mon Sep 17 00:00:00 2001 From: steviez Date: Thu, 24 Oct 2024 16:28:00 -0500 Subject: [PATCH 6/6] alphabetical order --- accounts-db/src/accounts_db.rs | 8 +++---- validator/src/cli/thread_args.rs | 38 ++++++++++++++++---------------- validator/src/main.rs | 4 ++-- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/accounts-db/src/accounts_db.rs b/accounts-db/src/accounts_db.rs index 56754ff9711275..d69cd895ccafd8 100644 --- a/accounts-db/src/accounts_db.rs +++ b/accounts-db/src/accounts_db.rs @@ -511,8 +511,8 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig { scan_filter_for_shrinking: ScanFilter::OnlyAbnormalWithVerify, enable_experimental_accumulator_hash: false, num_clean_threads: None, - num_hash_threads: None, num_foreground_threads: None, + num_hash_threads: None, }; pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig { index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS), @@ -531,8 +531,8 @@ pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig scan_filter_for_shrinking: ScanFilter::OnlyAbnormalWithVerify, enable_experimental_accumulator_hash: false, num_clean_threads: None, - num_hash_threads: None, num_foreground_threads: None, + num_hash_threads: None, }; pub type BinnedHashData = Vec>; @@ -646,10 +646,10 @@ pub struct AccountsDbConfig { pub enable_experimental_accumulator_hash: bool, /// Number of threads for background cleaning operations (`thread_pool_clean') pub num_clean_threads: Option, - /// Number of threads for background accounts hashing (`thread_pool_hash`) - pub num_hash_threads: Option, /// Number of threads for foreground operations (`thread_pool`) pub num_foreground_threads: Option, + /// Number of threads for background accounts hashing (`thread_pool_hash`) + pub num_hash_threads: Option, } #[cfg(not(test))] diff --git a/validator/src/cli/thread_args.rs b/validator/src/cli/thread_args.rs index 72cda6d6e71d4b..314e82846b5e2c 100644 --- a/validator/src/cli/thread_args.rs +++ b/validator/src/cli/thread_args.rs @@ -11,8 +11,8 @@ use { // Need this struct to provide &str whose lifetime matches that of the CLAP Arg's pub struct DefaultThreadArgs { pub accounts_db_clean_threads: String, - pub accounts_db_hash_threads: String, pub accounts_db_foreground_threads: String, + pub accounts_db_hash_threads: String, pub accounts_index_flush_threads: String, pub ip_echo_server_threads: String, pub replay_forks_threads: String, @@ -25,9 +25,9 @@ impl Default for DefaultThreadArgs { fn default() -> Self { Self { accounts_db_clean_threads: AccountsDbCleanThreadsArg::bounded_default().to_string(), - accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), accounts_db_foreground_threads: AccountsDbForegroundThreadsArg::bounded_default() .to_string(), + accounts_db_hash_threads: AccountsDbHashThreadsArg::bounded_default().to_string(), accounts_index_flush_threads: AccountsIndexFlushThreadsArg::bounded_default() .to_string(), ip_echo_server_threads: IpEchoServerThreadsArg::bounded_default().to_string(), @@ -43,8 +43,8 @@ impl Default for DefaultThreadArgs { pub fn thread_args<'a>(defaults: &DefaultThreadArgs) -> Vec> { vec![ new_thread_arg::(&defaults.accounts_db_clean_threads), - new_thread_arg::(&defaults.accounts_db_hash_threads), new_thread_arg::(&defaults.accounts_db_foreground_threads), + new_thread_arg::(&defaults.accounts_db_hash_threads), new_thread_arg::(&defaults.accounts_db_foreground_threads), new_thread_arg::(&defaults.ip_echo_server_threads), new_thread_arg::(&defaults.replay_forks_threads), @@ -67,8 +67,8 @@ fn new_thread_arg<'a, T: ThreadArg>(default: &str) -> Arg<'_, 'a> { pub struct NumThreadConfig { pub accounts_db_clean_threads: NonZeroUsize, - pub accounts_db_hash_threads: NonZeroUsize, pub accounts_db_foreground_threads: NonZeroUsize, + pub accounts_db_hash_threads: NonZeroUsize, pub accounts_index_flush_threads: NonZeroUsize, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, @@ -84,14 +84,14 @@ pub fn parse_num_threads_args(matches: &ArgMatches) -> NumThreadConfig { AccountsDbCleanThreadsArg::NAME, NonZeroUsize ), - accounts_db_hash_threads: value_t_or_exit!( + accounts_db_foreground_threads: value_t_or_exit!( matches, - AccountsDbHashThreadsArg::NAME, + AccountsDbForegroundThreadsArg::NAME, NonZeroUsize ), - accounts_db_foreground_threads: value_t_or_exit!( + accounts_db_hash_threads: value_t_or_exit!( matches, - AccountsDbForegroundThreadsArg::NAME, + AccountsDbHashThreadsArg::NAME, NonZeroUsize ), accounts_index_flush_threads: value_t_or_exit!( @@ -166,17 +166,6 @@ impl ThreadArg for AccountsDbCleanThreadsArg { } } -struct AccountsDbHashThreadsArg; -impl ThreadArg for AccountsDbHashThreadsArg { - const NAME: &'static str = "accounts_db_hash_threads"; - const LONG_NAME: &'static str = "accounts-db-hash-threads"; - const HELP: &'static str = "Number of threads to use for background accounts hashing"; - - fn default() -> usize { - accounts_db::default_num_hash_threads().get() - } -} - struct AccountsDbForegroundThreadsArg; impl ThreadArg for AccountsDbForegroundThreadsArg { const NAME: &'static str = "accounts_db_foreground_threads"; @@ -188,6 +177,17 @@ impl ThreadArg for AccountsDbForegroundThreadsArg { } } +struct AccountsDbHashThreadsArg; +impl ThreadArg for AccountsDbHashThreadsArg { + const NAME: &'static str = "accounts_db_hash_threads"; + const LONG_NAME: &'static str = "accounts-db-hash-threads"; + const HELP: &'static str = "Number of threads to use for background accounts hashing"; + + fn default() -> usize { + accounts_db::default_num_hash_threads().get() + } +} + struct AccountsIndexFlushThreadsArg; impl ThreadArg for AccountsIndexFlushThreadsArg { const NAME: &'static str = "accounts_index_flush_threads"; diff --git a/validator/src/main.rs b/validator/src/main.rs index ca152f4b09d71f..5d218eedd4a9f4 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -911,8 +911,8 @@ pub fn main() { let cli::thread_args::NumThreadConfig { accounts_db_clean_threads, - accounts_db_hash_threads, accounts_db_foreground_threads, + accounts_db_hash_threads, accounts_index_flush_threads, ip_echo_server_threads, replay_forks_threads, @@ -1317,8 +1317,8 @@ pub fn main() { enable_experimental_accumulator_hash: matches .is_present("accounts_db_experimental_accumulator_hash"), num_clean_threads: Some(accounts_db_clean_threads), - num_hash_threads: Some(accounts_db_hash_threads), num_foreground_threads: Some(accounts_db_foreground_threads), + num_hash_threads: Some(accounts_db_hash_threads), ..AccountsDbConfig::default() };