diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000000..f4bc0528e11b2d --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,25 @@ +version: 2.1 + +orbs: + aws-ecr: circleci/aws-ecr@8.1.3 + +executors: + custom: + machine: + image: ubuntu-2204:2022.04.2 + resource_class: xlarge + +workflows: + run-pipeline: + jobs: + - aws-ecr/build-and-push-image: + executor: custom + dockerfile: _step-finance/Dockerfile + repo: step-solana + region: eu-north-1 + tag: ${CIRCLE_BRANCH},${CIRCLE_BRANCH}-<< pipeline.number >>,${CIRCLE_BRANCH}-<< pipeline.number >>-${CIRCLE_SHA1} + context: aws-account + filters: + branches: + only: + - step-release diff --git a/Cargo.lock b/Cargo.lock index 3f77b6a525918b..61b77e1d72fa4a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -270,6 +270,7 @@ dependencies = [ "solana-send-transaction-service", "solana-storage-bigtable", "solana-streamer", + "solana-svm", "solana-test-validator", "solana-tpu-client", "solana-unified-scheduler-pool", @@ -6368,6 +6369,7 @@ dependencies = [ "solana-rpc", "solana-runtime", "solana-sdk", + "solana-svm", "solana-transaction-status", "thiserror", "tokio", @@ -7482,6 +7484,7 @@ dependencies = [ "rustc_version 0.4.0", "serde", "serde_derive", + "serde_json", "solana-bpf-loader-program", "solana-compute-budget", "solana-compute-budget-program", @@ -7541,6 +7544,7 @@ dependencies = [ "solana-runtime", "solana-sdk", "solana-streamer", + "solana-svm", "solana-tpu-client", "tokio", ] diff --git a/_step-finance/Dockerfile b/_step-finance/Dockerfile new file mode 100644 index 00000000000000..2edf8d4eb77ac3 --- /dev/null +++ b/_step-finance/Dockerfile @@ -0,0 +1,14 @@ +FROM rust:1.73-bullseye as builder +RUN apt-get update +RUN apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler +WORKDIR /solana-build +COPY . . +RUN cargo build --release + +FROM debian:bullseye-20230522 as runtime +RUN apt-get update +RUN apt-get install -y bzip2 +COPY --from=builder --chmod=700 /solana-build/target/release /usr/local/bin + +WORKDIR /solana +ENTRYPOINT ["solana-test-validator"] diff --git a/_step-finance/Dockerfile.dockerignore b/_step-finance/Dockerfile.dockerignore new file mode 100644 index 00000000000000..fdab7b85601aec --- /dev/null +++ b/_step-finance/Dockerfile.dockerignore @@ -0,0 +1,2 @@ +target/ +_step-finance/ diff --git a/cli/src/cluster_query.rs b/cli/src/cluster_query.rs index 46067f29d38eb1..07fc7f99787489 100644 --- a/cli/src/cluster_query.rs +++ b/cli/src/cluster_query.rs @@ -2160,6 +2160,8 @@ pub fn process_transaction_history( block_time, slot, transaction: transaction_with_meta, + index_in_block: _, + slot_second_idx: _, } = confirmed_transaction; let decoded_transaction = diff --git a/cli/src/wallet.rs b/cli/src/wallet.rs index 66a19913f6a011..39ba5cd5996a30 100644 --- a/cli/src/wallet.rs +++ b/cli/src/wallet.rs @@ -771,6 +771,8 @@ pub fn process_confirm( block_time, slot, transaction: transaction_with_meta, + index_in_block: _, + slot_second_idx: _, } = confirmed_transaction; let decoded_transaction = diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs index 0ca1304a4560f5..4b88e1397b8daf 100644 --- a/core/src/banking_stage/committer.rs +++ b/core/src/banking_stage/committer.rs @@ -6,7 +6,10 @@ use { }, solana_measure::measure_us, solana_runtime::{ - bank::{Bank, ExecutedTransactionCounts, TransactionBalancesSet}, + bank::{ + Bank, ExecutedTransactionCounts, TransactionBalancesSet, TransactionDatumSet, + TransactionOwnersSet, + }, bank_utils, prioritization_fee_cache::PrioritizationFeeCache, transaction_batch::TransactionBatch, @@ -15,6 +18,7 @@ use { solana_sdk::{hash::Hash, pubkey::Pubkey, saturating_add_assign}, solana_svm::{ account_loader::TransactionLoadResult, + program_inclusions::PreOrPostDatum, transaction_results::{TransactionExecutionResult, TransactionResults}, }, solana_transaction_status::{ @@ -35,6 +39,7 @@ pub enum CommitTransactionDetails { #[derive(Default)] pub(super) struct PreBalanceInfo { pub native: Vec>, + pub datum: Vec>>>, pub token: Vec>, pub mint_decimals: HashMap, } @@ -154,7 +159,8 @@ impl Committer { ) { if let Some(transaction_status_sender) = &self.transaction_status_sender { let txs = batch.sanitized_transactions().to_vec(); - let post_balances = bank.collect_balances(batch); + let (post_balances, post_datum, owners) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum); let post_token_balances = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals); let mut transaction_index = starting_transaction_index.unwrap_or_default(); @@ -179,6 +185,8 @@ impl Committer { std::mem::take(&mut pre_balance_info.native), post_balances, ), + TransactionOwnersSet { owners }, + TransactionDatumSet::new(std::mem::take(&mut pre_balance_info.datum), post_datum), TransactionTokenBalancesSet::new( std::mem::take(&mut pre_balance_info.token), post_token_balances, diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs index e87b6374b98ed0..6a07bb6af68f86 100644 --- a/core/src/banking_stage/consumer.rs +++ b/core/src/banking_stage/consumer.rs @@ -33,6 +33,7 @@ use { }, solana_svm::{ account_loader::{validate_fee_payer, TransactionCheckResult}, + program_inclusions::PreOrPostDatum, transaction_error_metrics::TransactionErrorMetrics, transaction_processor::{ExecutionRecordingConfig, TransactionProcessingConfig}, }, @@ -576,7 +577,8 @@ impl Consumer { // If the extra meta-data services are enabled for RPC, collect the // pre-balances for native and token programs. if transaction_status_sender_enabled { - pre_balance_info.native = bank.collect_balances(batch); + (pre_balance_info.native, pre_balance_info.datum, ..) = + bank.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum); pre_balance_info.token = collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals) } diff --git a/core/src/validator.rs b/core/src/validator.rs index d89520c0c83163..d8172cad59a99f 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -533,6 +533,7 @@ impl Validator { let mut bank_notification_senders = Vec::new(); let exit = Arc::new(AtomicBool::new(false)); + let inclusions_copy = config.runtime_config.program_datum_inclusions.clone(); let geyser_plugin_service = if let Some(geyser_plugin_config_files) = &config.on_start_geyser_plugin_config_files { @@ -545,6 +546,7 @@ impl Validator { confirmed_bank_receiver, geyser_plugin_config_files, rpc_to_plugin_manager_receiver_and_exit, + inclusions_copy, ) .map_err(|err| format!("Failed to load the Geyser plugin: {err:?}"))?, ) diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml index a7b02f8d593a8d..290396ba94a3b0 100644 --- a/geyser-plugin-manager/Cargo.toml +++ b/geyser-plugin-manager/Cargo.toml @@ -26,6 +26,7 @@ solana-metrics = { workspace = true } solana-rpc = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-svm = { workspace = true } solana-transaction-status = { workspace = true } thiserror = { workspace = true } tokio = { workspace = true } diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs index d5521c9ad41e19..9e441d6a36dde8 100644 --- a/geyser-plugin-manager/src/geyser_plugin_manager.rs +++ b/geyser-plugin-manager/src/geyser_plugin_manager.rs @@ -3,9 +3,11 @@ use { jsonrpc_core::{ErrorCode, Result as JsonRpcResult}, libloading::Library, log::*, + solana_svm::program_inclusions::{load_datum_program_inclusions, ProgramDatumInclusions}, std::{ ops::{Deref, DerefMut}, path::Path, + sync::{Arc, RwLock}, }, tokio::sync::oneshot::Sender as OneShotSender, }; @@ -47,13 +49,15 @@ impl DerefMut for LoadedGeyserPlugin { pub struct GeyserPluginManager { pub plugins: Vec, libs: Vec, + inclusions: Arc>, } impl GeyserPluginManager { - pub fn new() -> Self { + pub fn new(inclusions: Arc>) -> Self { GeyserPluginManager { plugins: Vec::default(), libs: Vec::default(), + inclusions, } } @@ -239,6 +243,13 @@ impl GeyserPluginManager { Ok(()) => { self.plugins.push(new_plugin); self.libs.push(new_lib); + + // Reload datum inclusions + log::info!("Reloading datum inclusions"); + let mut inclusions_write_lock = self.inclusions.write().unwrap(); + *inclusions_write_lock = + load_datum_program_inclusions(&Some(vec![config_file.into()])); + log::info!("Reloaded datum inclusions"); } // On failure, return error @@ -487,7 +498,9 @@ mod tests { #[test] fn test_geyser_reload() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); // No plugins are loaded, this should fail let mut plugin_manager_lock = plugin_manager.write().unwrap(); @@ -528,7 +541,9 @@ mod tests { #[test] fn test_plugin_list() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); let mut plugin_manager_lock = plugin_manager.write().unwrap(); // Load two plugins @@ -552,7 +567,9 @@ mod tests { #[test] fn test_plugin_load_unload() { // Initialize empty manager - let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new())); + let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))); let mut plugin_manager_lock = plugin_manager.write().unwrap(); // Load rpc call diff --git a/geyser-plugin-manager/src/geyser_plugin_service.rs b/geyser-plugin-manager/src/geyser_plugin_service.rs index ff3e050dc4b391..edad9c7cbced5f 100644 --- a/geyser-plugin-manager/src/geyser_plugin_service.rs +++ b/geyser-plugin-manager/src/geyser_plugin_service.rs @@ -17,6 +17,7 @@ use { optimistically_confirmed_bank_tracker::SlotNotification, transaction_notifier_interface::TransactionNotifierArc, }, + solana_svm::program_inclusions::ProgramDatumInclusions, std::{ path::{Path, PathBuf}, sync::{ @@ -55,8 +56,14 @@ impl GeyserPluginService { pub fn new( confirmed_bank_receiver: Receiver, geyser_plugin_config_files: &[PathBuf], + inclusions: Arc>, ) -> Result { - Self::new_with_receiver(confirmed_bank_receiver, geyser_plugin_config_files, None) + Self::new_with_receiver( + confirmed_bank_receiver, + geyser_plugin_config_files, + None, + inclusions, + ) } pub fn new_with_receiver( @@ -66,12 +73,13 @@ impl GeyserPluginService { Receiver, Arc, )>, + inclusions: Arc>, ) -> Result { info!( "Starting GeyserPluginService from config files: {:?}", geyser_plugin_config_files ); - let mut plugin_manager = GeyserPluginManager::new(); + let mut plugin_manager = GeyserPluginManager::new(inclusions); for geyser_plugin_config_file in geyser_plugin_config_files { Self::load_plugin(&mut plugin_manager, geyser_plugin_config_file)?; diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs index 30f1d77a68043e..bce0ace070356d 100644 --- a/ledger-tool/src/ledger_utils.rs +++ b/ledger-tool/src/ledger_utils.rs @@ -271,9 +271,16 @@ pub fn load_and_process_ledger( let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded(); drop(confirmed_bank_sender); - let geyser_service = - GeyserPluginService::new(confirmed_bank_receiver, &geyser_config_files) - .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?; + let inclusions_copy = process_options + .runtime_config + .program_datum_inclusions + .clone(); + let geyser_service = GeyserPluginService::new( + confirmed_bank_receiver, + &geyser_config_files, + inclusions_copy, + ) + .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?; ( geyser_service.get_accounts_update_notifier(), geyser_service.get_transaction_notifier(), diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs index 67330d5a949cbb..b7fc62a4e2a505 100644 --- a/ledger/src/blockstore_processor.rs +++ b/ledger/src/blockstore_processor.rs @@ -36,7 +36,7 @@ use { solana_rayon_threadlimit::{get_max_thread_count, get_thread_count}, solana_runtime::{ accounts_background_service::{AbsRequestSender, SnapshotRequestKind}, - bank::{Bank, TransactionBalancesSet}, + bank::{Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet}, bank_forks::{BankForks, SetRootError}, bank_utils, commitment::VOTE_THRESHOLD_SIZE, @@ -167,7 +167,7 @@ pub fn execute_batch( vec![] }; - let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions( + let (tx_results, balances, datum, owners) = batch.bank().load_execute_and_commit_transactions( batch, MAX_PROCESSING_AGE, transaction_status_sender.is_some(), @@ -233,6 +233,8 @@ pub fn execute_batch( transactions, execution_results, balances, + owners, + datum, token_balances, rent_debits, transaction_indexes.to_vec(), @@ -2142,6 +2144,8 @@ pub struct TransactionStatusBatch { pub transactions: Vec, pub execution_results: Vec>, pub balances: TransactionBalancesSet, + pub owners: TransactionOwnersSet, + pub datum: TransactionDatumSet, pub token_balances: TransactionTokenBalancesSet, pub rent_debits: Vec, pub transaction_indexes: Vec, @@ -2159,6 +2163,8 @@ impl TransactionStatusSender { transactions: Vec, execution_results: Vec, balances: TransactionBalancesSet, + owners: TransactionOwnersSet, + datum: TransactionDatumSet, token_balances: TransactionTokenBalancesSet, rent_debits: Vec, transaction_indexes: Vec, @@ -2178,6 +2184,8 @@ impl TransactionStatusSender { }) .collect(), balances, + owners, + datum, token_balances, rent_debits, transaction_indexes, @@ -4279,6 +4287,8 @@ pub mod tests { .. }, _balances, + _datums, + _owners, ) = batch.bank().load_execute_and_commit_transactions( &batch, MAX_PROCESSING_AGE, diff --git a/rpc-client/src/mock_sender.rs b/rpc-client/src/mock_sender.rs index 9730a6ff24a983..dc9f5ac891e83b 100644 --- a/rpc-client/src/mock_sender.rs +++ b/rpc-client/src/mock_sender.rs @@ -193,6 +193,9 @@ impl RpcSender for MockSender { fee: 0, pre_balances: vec![499999999999999950, 50, 1], post_balances: vec![499999999999999950, 50, 1], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: OptionSerializer::None, log_messages: OptionSerializer::None, pre_token_balances: OptionSerializer::None, @@ -204,6 +207,8 @@ impl RpcSender for MockSender { }), }, block_time: Some(1628633791), + index_in_block: 0, + slot_second_idx: None, })?, "getTransactionCount" => json![1234], "getSlot" => json![0], diff --git a/rpc/src/rpc.rs b/rpc/src/rpc.rs index 1aedda8d0920ce..65c3b29063ebea 100644 --- a/rpc/src/rpc.rs +++ b/rpc/src/rpc.rs @@ -1600,7 +1600,7 @@ impl JsonRpcRequestProcessor { let encode_transaction = |confirmed_tx_with_meta: ConfirmedTransactionWithStatusMeta| -> Result { - Ok(confirmed_tx_with_meta.encode(encoding, max_supported_transaction_version).map_err(RpcCustomError::from)?) + Ok(confirmed_tx_with_meta.encode(encoding, max_supported_transaction_version, 0).map_err(RpcCustomError::from)?) }; match confirmed_transaction.unwrap_or(None) { diff --git a/rpc/src/transaction_status_service.rs b/rpc/src/transaction_status_service.rs index 34e9b7fdc6f8b2..d3a162a3d5e188 100644 --- a/rpc/src/transaction_status_service.rs +++ b/rpc/src/transaction_status_service.rs @@ -69,6 +69,8 @@ impl TransactionStatusService { bank, transactions, execution_results, + owners, + datum, balances, token_balances, rent_debits, @@ -80,6 +82,9 @@ impl TransactionStatusService { execution_result, pre_balances, post_balances, + owners, + pre_datum, + post_datum, pre_token_balances, post_token_balances, rent_debits, @@ -89,6 +94,9 @@ impl TransactionStatusService { execution_results, balances.pre_balances, balances.post_balances, + owners.owners, + datum.pre_datum, + datum.post_datum, token_balances.pre_token_balances, token_balances.post_token_balances, rent_debits, @@ -131,6 +139,9 @@ impl TransactionStatusService { fee, pre_balances, post_balances, + post_owners: Some(owners), + pre_datum: Some(pre_datum), + post_datum: Some(post_datum), inner_instructions, log_messages, pre_token_balances, @@ -193,6 +204,7 @@ impl TransactionStatusService { #[cfg(test)] pub(crate) mod tests { + use { super::*, crate::transaction_notifier_interface::TransactionNotifier, @@ -202,7 +214,9 @@ pub(crate) mod tests { parse_account_data::SplTokenAdditionalData, parse_token::token_amount_to_ui_amount_v2, }, solana_ledger::{genesis_utils::create_genesis_config, get_tmp_ledger_path_auto_delete}, - solana_runtime::bank::{Bank, TransactionBalancesSet}, + solana_runtime::bank::{ + Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet, + }, solana_sdk::{ account_utils::StateMut, clock::Slot, @@ -374,6 +388,11 @@ pub(crate) mod tests { let transaction_index: usize = bank.transaction_count().try_into().unwrap(); let transaction_status_batch = TransactionStatusBatch { bank, + owners: TransactionOwnersSet { owners: vec![] }, + datum: TransactionDatumSet { + post_datum: vec![vec![Some(vec![0x69])]], + pre_datum: vec![vec![Some(vec![0x04, 0x20])]], + }, transactions: vec![transaction], execution_results: vec![transaction_result], balances, diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index 5d5b54eb384ed9..7fd838dac28dba 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -164,6 +164,7 @@ use { }, account_overrides::AccountOverrides, nonce_info::NoncePartial, + program_inclusions::{PreOrPostDatum, ProgramDatumInclusions}, transaction_error_metrics::TransactionErrorMetrics, transaction_processing_callback::TransactionProcessingCallback, transaction_processor::{ @@ -381,6 +382,27 @@ impl TransactionBalancesSet { } pub type TransactionBalances = Vec>; +pub struct TransactionDatumSet { + pub pre_datum: TransactionDatum, + pub post_datum: TransactionDatum, +} + +impl TransactionDatumSet { + pub fn new(pre_datum: TransactionDatum, post_datum: TransactionDatum) -> Self { + Self { + pre_datum, + post_datum, + } + } +} +pub type TransactionDatum = Vec>>>; + +pub struct TransactionOwnersSet { + pub owners: TransactionOwners, +} + +pub type TransactionOwners = Vec>>; + #[cfg_attr(feature = "frozen-abi", derive(AbiExample, AbiEnumVisitor))] #[derive(Serialize, Deserialize, Debug, PartialEq, Eq)] pub enum TransactionLogCollectorFilter { @@ -591,6 +613,7 @@ impl PartialEq for Bank { accounts_data_size_delta_off_chain: _, incremental_snapshot_persistence: _, epoch_reward_status: _, + program_datum_inclusions: _, transaction_processor: _, check_program_modification_slot: _, collector_fee_details: _, @@ -874,6 +897,8 @@ pub struct Bank { check_program_modification_slot: bool, + /// programs that will have their account data sent to geyser + pub program_datum_inclusions: Arc>, /// Collected fee details collector_fee_details: RwLock, @@ -997,6 +1022,7 @@ impl Bank { accounts_data_size_delta_on_chain: AtomicI64::new(0), accounts_data_size_delta_off_chain: AtomicI64::new(0), epoch_reward_status: EpochRewardStatus::default(), + program_datum_inclusions: Arc::new(RwLock::new(ProgramDatumInclusions::default())), transaction_processor: TransactionBatchProcessor::default(), check_program_modification_slot: false, collector_fee_details: RwLock::new(CollectorFeeDetails::default()), @@ -1040,6 +1066,7 @@ impl Bank { ); let accounts = Accounts::new(Arc::new(accounts_db)); let mut bank = Self::default_with_accounts(accounts); + bank.program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); bank.ancestors = Ancestors::from(vec![bank.slot()]); bank.compute_budget = runtime_config.compute_budget; bank.transaction_account_lock_limit = runtime_config.transaction_account_lock_limit; @@ -1243,6 +1270,7 @@ impl Bank { accounts_data_size_delta_on_chain: AtomicI64::new(0), accounts_data_size_delta_off_chain: AtomicI64::new(0), epoch_reward_status: parent.epoch_reward_status.clone(), + program_datum_inclusions: parent.program_datum_inclusions.clone(), transaction_processor, check_program_modification_slot: false, collector_fee_details: RwLock::new(CollectorFeeDetails::default()), @@ -1575,6 +1603,7 @@ impl Bank { )); info!("Loading Stakes took: {stakes_time}"); let stakes_accounts_load_duration = now.elapsed(); + let program_datum_inclusions = runtime_config.program_datum_inclusions.clone(); let mut bank = Self { skipped_rewrites: Mutex::default(), incremental_snapshot_persistence: fields.incremental_snapshot_persistence, @@ -1635,6 +1664,7 @@ impl Bank { epoch_reward_status: EpochRewardStatus::default(), transaction_processor: TransactionBatchProcessor::default(), check_program_modification_slot: false, + program_datum_inclusions, // collector_fee_details is not serialized to snapshot collector_fee_details: RwLock::new(CollectorFeeDetails::default()), compute_budget: runtime_config.compute_budget, @@ -3616,6 +3646,31 @@ impl Bank { balances } + pub fn collect_balances_and_datum( + &self, + batch: &TransactionBatch, + pre_or_post: PreOrPostDatum, + ) -> (TransactionBalances, TransactionDatum, TransactionOwners) { + let mut balances: TransactionBalances = vec![]; + let mut datum: TransactionDatum = vec![]; + let mut owners: TransactionOwners = vec![]; + for transaction in batch.sanitized_transactions() { + let mut transaction_balances: Vec = vec![]; + let mut transaction_datum: Vec>> = vec![]; + let mut transaction_owners: Vec> = vec![]; + for account_key in transaction.message().account_keys().iter() { + let (balance, data, owner) = self.get_balance_and_data(account_key, &pre_or_post); + transaction_balances.push(balance); + transaction_datum.push(data); + transaction_owners.push(owner); + } + balances.push(transaction_balances); + datum.push(transaction_datum); + owners.push(transaction_owners); + } + (balances, datum, owners) + } + pub fn load_and_execute_transactions( &self, batch: &TransactionBatch, @@ -4776,11 +4831,16 @@ impl Bank { recording_config: ExecutionRecordingConfig, timings: &mut ExecuteTimings, log_messages_bytes_limit: Option, - ) -> (TransactionResults, TransactionBalancesSet) { - let pre_balances = if collect_balances { - self.collect_balances(batch) + ) -> ( + TransactionResults, + TransactionBalancesSet, + TransactionDatumSet, + TransactionOwnersSet, + ) { + let (pre_balances, pre_datum, ..) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum) } else { - vec![] + (vec![], vec![], vec![]) }; let LoadAndExecuteTransactionsOutput { @@ -4824,14 +4884,16 @@ impl Bank { }, timings, ); - let post_balances = if collect_balances { - self.collect_balances(batch) + let (post_balances, post_datum, owners) = if collect_balances { + self.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum) } else { - vec![] + (vec![], vec![], vec![]) }; ( results, TransactionBalancesSet::new(pre_balances, post_balances), + TransactionDatumSet::new(pre_datum, post_datum), + TransactionOwnersSet { owners }, ) } @@ -4927,6 +4989,24 @@ impl Bank { pub fn read_balance(account: &AccountSharedData) -> u64 { account.lamports() } + + pub fn read_data( + &self, + account: &AccountSharedData, + pre_or_post: &PreOrPostDatum, + ) -> Option> { + let data = account.data(); + let owner = account.owner(); + let inclusions_lock = self.program_datum_inclusions.read().unwrap(); + let inclusion = inclusions_lock.get(owner)?; + let include_data = inclusion.can_include_datum(pre_or_post, &data); + + if !include_data { + None + } else { + Some(data.to_vec()) + } + } /// Each program would need to be able to introspect its own state /// this is hard-coded to the Budget language pub fn get_balance(&self, pubkey: &Pubkey) -> u64 { @@ -4934,6 +5014,21 @@ impl Bank { .map(|x| Self::read_balance(&x)) .unwrap_or(0) } + pub fn get_balance_and_data( + &self, + pubkey: &Pubkey, + pre_or_post: &PreOrPostDatum, + ) -> (u64, Option>, Option) { + self.get_account(pubkey) + .map(|x| { + ( + Self::read_balance(&x), + Self::read_data(self, &x, pre_or_post), + Some(*x.owner()), + ) + }) + .unwrap_or((0, None, None)) + } /// Compute all the parents of the bank in order pub fn parents(&self) -> Vec> { diff --git a/runtime/src/bank/tests.rs b/runtime/src/bank/tests.rs index 3da66902e95ae8..87ea40d7191744 100644 --- a/runtime/src/bank/tests.rs +++ b/runtime/src/bank/tests.rs @@ -5842,7 +5842,7 @@ fn test_pre_post_transaction_balances() { let txs = vec![tx0, tx1, tx2]; let lock_result = bank0.prepare_batch_for_tests(txs); - let (transaction_results, transaction_balances_set) = bank0 + let (transaction_results, transaction_balances_set, _) = bank0 .load_execute_and_commit_transactions( &lock_result, MAX_PROCESSING_AGE, diff --git a/storage-bigtable/src/lib.rs b/storage-bigtable/src/lib.rs index 3af928a626d834..6d64b3340e105d 100644 --- a/storage-bigtable/src/lib.rs +++ b/storage-bigtable/src/lib.rs @@ -250,6 +250,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/storage-proto/src/convert.rs b/storage-proto/src/convert.rs index 8315bcf99a4dac..8ce5f34415bf23 100644 --- a/storage-proto/src/convert.rs +++ b/storage-proto/src/convert.rs @@ -405,6 +405,7 @@ impl From for generated::TransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; let err = match status { Ok(()) => None, @@ -558,6 +559,9 @@ impl TryFrom for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances, diff --git a/storage-proto/src/lib.rs b/storage-proto/src/lib.rs index 0832690f1e2b88..09ccf0014e7f1e 100644 --- a/storage-proto/src/lib.rs +++ b/storage-proto/src/lib.rs @@ -200,6 +200,9 @@ impl From for TransactionStatusMeta { fee, pre_balances, post_balances, + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions, log_messages, pre_token_balances: pre_token_balances @@ -231,6 +234,7 @@ impl TryFrom for StoredTransactionStatusMeta { loaded_addresses, return_data, compute_units_consumed, + .. } = value; if !loaded_addresses.is_empty() { diff --git a/svm/Cargo.toml b/svm/Cargo.toml index c53f13c91c2706..6564f553747949 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -16,6 +16,7 @@ percentage = { workspace = true } qualifier_attr = { workspace = true } serde = { workspace = true, features = ["rc"] } serde_derive = { workspace = true } +serde_json = { workspace = true } solana-bpf-loader-program = { workspace = true } solana-compute-budget = { workspace = true } solana-frozen-abi = { workspace = true, optional = true } diff --git a/svm/src/lib.rs b/svm/src/lib.rs index 201bf671a1be49..76a8c6d09b2a4a 100644 --- a/svm/src/lib.rs +++ b/svm/src/lib.rs @@ -6,6 +6,7 @@ pub mod account_overrides; pub mod account_rent_state; pub mod message_processor; pub mod nonce_info; +pub mod program_inclusions; pub mod program_loader; pub mod rollback_accounts; pub mod runtime_config; diff --git a/svm/src/program_inclusions.rs b/svm/src/program_inclusions.rs new file mode 100644 index 00000000000000..a70a82bcf7e55d --- /dev/null +++ b/svm/src/program_inclusions.rs @@ -0,0 +1,74 @@ +use std::{collections::HashMap, fs, path::PathBuf}; + +use serde::Deserialize; +use solana_sdk::pubkey::Pubkey; + +pub enum PreOrPostDatum { + PreDatum, + PostDatum, +} + +pub type ProgramDatumInclusions = HashMap; +pub type InclusionsFromConfig = HashMap; + +#[derive(Deserialize, Debug, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct DatumInclusion { + #[serde(default)] + pub pre: bool, + #[serde(default)] + pub post: bool, + #[serde(default)] + pub length_exclusions: Vec, +} + +impl DatumInclusion { + pub fn can_include_datum(&self, pre_or_post: &PreOrPostDatum, data: &[u8]) -> bool { + let allow_pre_post = match pre_or_post { + PreOrPostDatum::PreDatum => self.pre, + PreOrPostDatum::PostDatum => self.post, + }; + + if !allow_pre_post { + return false; + } + + !self.length_exclusions.contains(&data.len()) + } +} + +#[derive(serde::Deserialize)] +struct GeyserConfigFileWithInclusions { + #[serde(rename = "datumProgramInclusions")] + pub datum_program_inclusions: InclusionsFromConfig, +} + +pub fn load_datum_program_inclusions(paths: &Option>) -> ProgramDatumInclusions { + let mut datum_program_inclusions: ProgramDatumInclusions = HashMap::new(); + if let Some(paths) = paths { + for path in paths { + let file = fs::read(path); + if !file.is_ok() { + eprintln!("Unable to read JSON file: {:?} Skipping...", path); + continue; + } + let json = serde_json::from_slice::(&file.unwrap()); + if let Err(e) = json { + eprintln!("Unable to parse JSON file {:?}: {:?} Skipping...", path, e); + continue; + } + let inclusions_map = json.unwrap().datum_program_inclusions; + for (pubkey, inc) in inclusions_map.iter() { + let pk_parsed = pubkey.parse::().expect( + format!( + "Bad pubkey provided to datumProgramInclusions in geyser config file {:?}", + path + ) + .as_str(), + ); + datum_program_inclusions.insert(pk_parsed, inc.clone()); + } + } + } + datum_program_inclusions +} diff --git a/svm/src/runtime_config.rs b/svm/src/runtime_config.rs index 7e063bc30e5b5d..99656809c820b3 100644 --- a/svm/src/runtime_config.rs +++ b/svm/src/runtime_config.rs @@ -1,4 +1,6 @@ +use crate::program_inclusions::ProgramDatumInclusions; use solana_compute_budget::compute_budget::ComputeBudget; +use std::sync::{Arc, RwLock}; #[cfg(all(RUSTC_WITH_SPECIALIZATION, feature = "frozen-abi"))] impl ::solana_frozen_abi::abi_example::AbiExample for RuntimeConfig { @@ -14,4 +16,5 @@ pub struct RuntimeConfig { pub compute_budget: Option, pub log_messages_bytes_limit: Option, pub transaction_account_lock_limit: Option, + pub program_datum_inclusions: Arc>, } diff --git a/test-validator/Cargo.toml b/test-validator/Cargo.toml index 0068ddbfb2bf0a..da1489ee02210b 100644 --- a/test-validator/Cargo.toml +++ b/test-validator/Cargo.toml @@ -31,6 +31,7 @@ solana-rpc = { workspace = true } solana-rpc-client = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-svm = { workspace = true } solana-streamer = { workspace = true } solana-tpu-client = { workspace = true } tokio = { workspace = true, features = ["full"] } diff --git a/test-validator/src/lib.rs b/test-validator/src/lib.rs index da0d669b76393a..240432550a0d17 100644 --- a/test-validator/src/lib.rs +++ b/test-validator/src/lib.rs @@ -53,6 +53,7 @@ use { signature::{read_keypair_file, write_keypair_file, Keypair, Signer}, }, solana_streamer::socket::SocketAddrSpace, + solana_svm::program_inclusions::load_datum_program_inclusions, solana_tpu_client::tpu_client::{ DEFAULT_TPU_CONNECTION_POOL_SIZE, DEFAULT_TPU_ENABLE_UDP, DEFAULT_TPU_USE_QUIC, }, @@ -166,7 +167,9 @@ impl Default for TestValidatorGenesis { log_messages_bytes_limit: Option::::default(), transaction_account_lock_limit: Option::::default(), tpu_enable_udp: DEFAULT_TPU_ENABLE_UDP, - geyser_plugin_manager: Arc::new(RwLock::new(GeyserPluginManager::new())), + geyser_plugin_manager: Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new( + RwLock::new(Default::default()), + )))), admin_rpc_service_post_init: Arc::>>::default(), } @@ -953,6 +956,10 @@ impl TestValidator { ..AccountsDbConfig::default() }); + let program_datum_inclusions = Arc::new(RwLock::new(load_datum_program_inclusions( + &config.geyser_plugin_config_files, + ))); + let runtime_config = RuntimeConfig { compute_budget: config .compute_unit_limit @@ -962,6 +969,7 @@ impl TestValidator { }), log_messages_bytes_limit: config.log_messages_bytes_limit, transaction_account_lock_limit: config.transaction_account_lock_limit, + program_datum_inclusions, }; let mut validator_config = ValidatorConfig { diff --git a/transaction-status/src/lib.rs b/transaction-status/src/lib.rs index 6a5eb5fb8d6397..3fa4e1bdf08f1e 100644 --- a/transaction-status/src/lib.rs +++ b/transaction-status/src/lib.rs @@ -354,6 +354,12 @@ pub struct TransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>>, + pub post_datum: Option>>>, pub inner_instructions: Option>, pub log_messages: Option>, pub pre_token_balances: Option>, @@ -371,6 +377,9 @@ impl Default for TransactionStatusMeta { fee: 0, pre_balances: vec![], post_balances: vec![], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, @@ -392,6 +401,12 @@ pub struct UiTransactionStatusMeta { pub fee: u64, pub pre_balances: Vec, pub post_balances: Vec, + pub post_owners: Option>>, + //the first Option is for backward compat + //the inner Option is for if we filtered out the datum (exceeds max size) + //if account data is empty, that is "", not None + pub pre_datum: Option>>, + pub post_datum: Option>>, #[serde( default = "OptionSerializer::none", skip_serializing_if = "OptionSerializer::should_skip" @@ -468,6 +483,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| { @@ -501,6 +533,23 @@ impl UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: OptionSerializer::Skip, log_messages: OptionSerializer::Skip, pre_token_balances: meta @@ -531,6 +580,23 @@ impl From for UiTransactionStatusMeta { fee: meta.fee, pre_balances: meta.pre_balances, post_balances: meta.post_balances, + post_owners: meta.post_owners.map(|o| { + o.into_iter() + .map(|ko| ko.map(|pk| pk.to_string())) + .collect() + }), + pre_datum: meta.pre_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), + post_datum: meta.post_datum.map(|a| { + a.into_iter() + .map(|b| b.map(|c| BASE64_STANDARD.encode(c))) + .into_iter() + .collect() + }), inner_instructions: meta .inner_instructions .map(|ixs| ixs.into_iter().map(Into::into).collect()) @@ -1057,6 +1123,7 @@ impl ConfirmedTransactionWithStatusMeta { self, encoding: UiTransactionEncoding, max_supported_transaction_version: Option, + index_in_block: usize, ) -> Result { Ok(EncodedConfirmedTransactionWithStatusMeta { slot: self.slot, @@ -1066,6 +1133,8 @@ impl ConfirmedTransactionWithStatusMeta { true, )?, block_time: self.block_time, + index_in_block, + slot_second_idx: None, }) } @@ -1074,13 +1143,15 @@ impl ConfirmedTransactionWithStatusMeta { } } -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EncodedConfirmedTransactionWithStatusMeta { pub slot: Slot, #[serde(flatten)] pub transaction: EncodedTransactionWithStatusMeta, pub block_time: Option, + pub index_in_block: usize, + pub slot_second_idx: Option, } #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1544,6 +1615,9 @@ mod test { fee: 1234, pre_balances: vec![1, 2, 3], post_balances: vec![4, 5, 6], + post_owners: None, + pre_datum: None, + post_datum: None, inner_instructions: None, log_messages: None, pre_token_balances: None, diff --git a/validator/Cargo.toml b/validator/Cargo.toml index 6222435906a31d..c5dd42280ea95f 100644 --- a/validator/Cargo.toml +++ b/validator/Cargo.toml @@ -56,6 +56,7 @@ solana-rpc-client = { workspace = true } solana-rpc-client-api = { workspace = true } solana-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-svm = { workspace = true } solana-send-transaction-service = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } diff --git a/validator/src/main.rs b/validator/src/main.rs index 8467bcaa9d2570..b489440e858994 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -72,6 +72,7 @@ use { }, solana_send_transaction_service::send_transaction_service, solana_streamer::socket::SocketAddrSpace, + solana_svm::program_inclusions::load_datum_program_inclusions, solana_tpu_client::tpu_client::DEFAULT_TPU_ENABLE_UDP, std::{ collections::{HashSet, VecDeque}, @@ -1307,6 +1308,10 @@ pub fn main() { }; let starting_with_geyser_plugins: bool = on_start_geyser_plugin_config_files.is_some(); + let program_datum_inclusions = Arc::new(RwLock::new(load_datum_program_inclusions( + &on_start_geyser_plugin_config_files, + ))); + let rpc_bigtable_config = if matches.is_present("enable_rpc_bigtable_ledger_storage") || matches.is_present("enable_bigtable_ledger_upload") { @@ -1510,6 +1515,7 @@ pub fn main() { accounts_shrink_ratio, runtime_config: RuntimeConfig { log_messages_bytes_limit: value_of(&matches, "log_messages_bytes_limit"), + program_datum_inclusions, ..RuntimeConfig::default() }, staked_nodes_overrides: staked_nodes_overrides.clone(),