Step Finance patch

Co-authored-by: Dana <[email protected]> bump docker build version Add account owners array to messages Cleaner diff rename to post_owners slot_second_idx index in block Reload config with plugin add logs Merge pull request anza-xyz#12 from step-finance/SF-5242-slot-number Slot second idx -> DEV
step-finance · Oct 18, 2024 · 6eab71f · 6eab71f
1 parent 4cc7b90
commit 6eab71f
Show file tree

Hide file tree

Showing 31 changed files with 429 additions and 26 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -0,0 +1,25 @@
+version: 2.1
+
+orbs:
+  aws-ecr: circleci/[email protected]
+
+executors:
+  custom:
+    machine:
+      image: ubuntu-2204:2022.04.2
+    resource_class: xlarge
+
+workflows:
+  run-pipeline:
+    jobs:
+      - aws-ecr/build-and-push-image:
+          executor: custom
+          dockerfile: _step-finance/Dockerfile
+          repo: step-solana
+          region: eu-north-1
+          tag: ${CIRCLE_BRANCH},${CIRCLE_BRANCH}-<< pipeline.number >>,${CIRCLE_BRANCH}-<< pipeline.number >>-${CIRCLE_SHA1}
+          context: aws-account
+          filters:
+            branches:
+              only:
+                - step-release
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/_step-finance/Dockerfile b/_step-finance/Dockerfile
@@ -0,0 +1,14 @@
+FROM rust:1.73-bullseye as builder
+RUN apt-get update
+RUN apt-get install -y libssl-dev libudev-dev pkg-config zlib1g-dev llvm clang cmake make libprotobuf-dev protobuf-compiler
+WORKDIR /solana-build
+COPY . .
+RUN cargo build --release
+
+FROM debian:bullseye-20230522 as runtime
+RUN apt-get update
+RUN apt-get install -y bzip2
+COPY --from=builder --chmod=700 /solana-build/target/release /usr/local/bin
+
+WORKDIR /solana
+ENTRYPOINT ["solana-test-validator"]
diff --git a/_step-finance/Dockerfile.dockerignore b/_step-finance/Dockerfile.dockerignore
@@ -0,0 +1,2 @@
+target/
+_step-finance/
diff --git a/cli/src/cluster_query.rs b/cli/src/cluster_query.rs
@@ -2160,6 +2160,8 @@ pub fn process_transaction_history(
                             block_time,
                             slot,
                             transaction: transaction_with_meta,
+                            index_in_block: _,
+                            slot_second_idx: _,
                         } = confirmed_transaction;
 
                         let decoded_transaction =

diff --git a/cli/src/wallet.rs b/cli/src/wallet.rs
@@ -771,6 +771,8 @@ pub fn process_confirm(
                                 block_time,
                                 slot,
                                 transaction: transaction_with_meta,
+                                index_in_block: _,
+                                slot_second_idx: _,
                             } = confirmed_transaction;
 
                             let decoded_transaction =

diff --git a/core/src/banking_stage/committer.rs b/core/src/banking_stage/committer.rs
@@ -6,7 +6,10 @@ use {
     },
     solana_measure::measure_us,
     solana_runtime::{
-        bank::{Bank, ExecutedTransactionCounts, TransactionBalancesSet},
+        bank::{
+            Bank, ExecutedTransactionCounts, TransactionBalancesSet, TransactionDatumSet,
+            TransactionOwnersSet,
+        },
         bank_utils,
         prioritization_fee_cache::PrioritizationFeeCache,
         transaction_batch::TransactionBatch,
@@ -15,6 +18,7 @@ use {
     solana_sdk::{hash::Hash, pubkey::Pubkey, saturating_add_assign},
     solana_svm::{
         account_loader::TransactionLoadResult,
+        program_inclusions::PreOrPostDatum,
         transaction_results::{TransactionExecutionResult, TransactionResults},
     },
     solana_transaction_status::{
@@ -35,6 +39,7 @@ pub enum CommitTransactionDetails {
 #[derive(Default)]
 pub(super) struct PreBalanceInfo {
     pub native: Vec<Vec<u64>>,
+    pub datum: Vec<Vec<Option<Vec<u8>>>>,
     pub token: Vec<Vec<TransactionTokenBalance>>,
     pub mint_decimals: HashMap<Pubkey, u8>,
 }
@@ -154,7 +159,8 @@ impl Committer {
     ) {
         if let Some(transaction_status_sender) = &self.transaction_status_sender {
             let txs = batch.sanitized_transactions().to_vec();
-            let post_balances = bank.collect_balances(batch);
+            let (post_balances, post_datum, owners) =
+                bank.collect_balances_and_datum(batch, PreOrPostDatum::PostDatum);
             let post_token_balances =
                 collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals);
             let mut transaction_index = starting_transaction_index.unwrap_or_default();
@@ -179,6 +185,8 @@ impl Committer {
                     std::mem::take(&mut pre_balance_info.native),
                     post_balances,
                 ),
+                TransactionOwnersSet { owners },
+                TransactionDatumSet::new(std::mem::take(&mut pre_balance_info.datum), post_datum),
                 TransactionTokenBalancesSet::new(
                     std::mem::take(&mut pre_balance_info.token),
                     post_token_balances,

diff --git a/core/src/banking_stage/consumer.rs b/core/src/banking_stage/consumer.rs
@@ -32,6 +32,7 @@ use {
     },
     solana_svm::{
         account_loader::{validate_fee_payer, TransactionCheckResult},
+        program_inclusions::PreOrPostDatum,
         transaction_error_metrics::TransactionErrorMetrics,
         transaction_processor::{ExecutionRecordingConfig, TransactionProcessingConfig},
     },
@@ -566,7 +567,8 @@ impl Consumer {
             // If the extra meta-data services are enabled for RPC, collect the
             // pre-balances for native and token programs.
             if transaction_status_sender_enabled {
-                pre_balance_info.native = bank.collect_balances(batch);
+                (pre_balance_info.native, pre_balance_info.datum, ..) =
+                    bank.collect_balances_and_datum(batch, PreOrPostDatum::PreDatum);
                 pre_balance_info.token =
                     collect_token_balances(bank, batch, &mut pre_balance_info.mint_decimals)
             }

diff --git a/core/src/validator.rs b/core/src/validator.rs
@@ -533,6 +533,7 @@ impl Validator {
         let mut bank_notification_senders = Vec::new();
 
         let exit = Arc::new(AtomicBool::new(false));
+        let inclusions_copy = config.runtime_config.program_datum_inclusions.clone();
 
         let geyser_plugin_service =
             if let Some(geyser_plugin_config_files) = &config.on_start_geyser_plugin_config_files {
@@ -545,6 +546,7 @@ impl Validator {
                         confirmed_bank_receiver,
                         geyser_plugin_config_files,
                         rpc_to_plugin_manager_receiver_and_exit,
+                        inclusions_copy,
                     )
                     .map_err(|err| format!("Failed to load the Geyser plugin: {err:?}"))?,
                 )

diff --git a/geyser-plugin-manager/Cargo.toml b/geyser-plugin-manager/Cargo.toml
@@ -26,6 +26,7 @@ solana-metrics = { workspace = true }
 solana-rpc = { workspace = true }
 solana-runtime = { workspace = true }
 solana-sdk = { workspace = true }
+solana-svm = { workspace = true }
 solana-transaction-status = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }

diff --git a/geyser-plugin-manager/src/geyser_plugin_manager.rs b/geyser-plugin-manager/src/geyser_plugin_manager.rs
@@ -3,9 +3,11 @@ use {
     jsonrpc_core::{ErrorCode, Result as JsonRpcResult},
     libloading::Library,
     log::*,
+    solana_svm::program_inclusions::{load_datum_program_inclusions, ProgramDatumInclusions},
     std::{
         ops::{Deref, DerefMut},
         path::Path,
+        sync::{Arc, RwLock},
     },
     tokio::sync::oneshot::Sender as OneShotSender,
 };
@@ -47,13 +49,15 @@ impl DerefMut for LoadedGeyserPlugin {
 pub struct GeyserPluginManager {
     pub plugins: Vec<LoadedGeyserPlugin>,
     libs: Vec<Library>,
+    inclusions: Arc<RwLock<ProgramDatumInclusions>>,
 }
 
 impl GeyserPluginManager {
-    pub fn new() -> Self {
+    pub fn new(inclusions: Arc<RwLock<ProgramDatumInclusions>>) -> Self {
         GeyserPluginManager {
             plugins: Vec::default(),
             libs: Vec::default(),
+            inclusions,
         }
     }
 
@@ -239,6 +243,13 @@ impl GeyserPluginManager {
             Ok(()) => {
                 self.plugins.push(new_plugin);
                 self.libs.push(new_lib);
+
+                // Reload datum inclusions
+                log::info!("Reloading datum inclusions");
+                let mut inclusions_write_lock = self.inclusions.write().unwrap();
+                *inclusions_write_lock =
+                    load_datum_program_inclusions(&Some(vec![config_file.into()]));
+                log::info!("Reloaded datum inclusions");
             }
 
             // On failure, return error
@@ -487,7 +498,9 @@ mod tests {
     #[test]
     fn test_geyser_reload() {
         // Initialize empty manager
-        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new()));
+        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new(
+            RwLock::new(Default::default()),
+        ))));
 
         // No plugins are loaded, this should fail
         let mut plugin_manager_lock = plugin_manager.write().unwrap();
@@ -528,7 +541,9 @@ mod tests {
     #[test]
     fn test_plugin_list() {
         // Initialize empty manager
-        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new()));
+        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new(
+            RwLock::new(Default::default()),
+        ))));
         let mut plugin_manager_lock = plugin_manager.write().unwrap();
 
         // Load two plugins
@@ -552,7 +567,9 @@ mod tests {
     #[test]
     fn test_plugin_load_unload() {
         // Initialize empty manager
-        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new()));
+        let plugin_manager = Arc::new(RwLock::new(GeyserPluginManager::new(Arc::new(
+            RwLock::new(Default::default()),
+        ))));
         let mut plugin_manager_lock = plugin_manager.write().unwrap();
 
         // Load rpc call

diff --git a/geyser-plugin-manager/src/geyser_plugin_service.rs b/geyser-plugin-manager/src/geyser_plugin_service.rs
@@ -17,6 +17,7 @@ use {
         optimistically_confirmed_bank_tracker::SlotNotification,
         transaction_notifier_interface::TransactionNotifierArc,
     },
+    solana_svm::program_inclusions::ProgramDatumInclusions,
     std::{
         path::{Path, PathBuf},
         sync::{
@@ -55,8 +56,14 @@ impl GeyserPluginService {
     pub fn new(
         confirmed_bank_receiver: Receiver<SlotNotification>,
         geyser_plugin_config_files: &[PathBuf],
+        inclusions: Arc<RwLock<ProgramDatumInclusions>>,
     ) -> Result<Self, GeyserPluginServiceError> {
-        Self::new_with_receiver(confirmed_bank_receiver, geyser_plugin_config_files, None)
+        Self::new_with_receiver(
+            confirmed_bank_receiver,
+            geyser_plugin_config_files,
+            None,
+            inclusions,
+        )
     }
 
     pub fn new_with_receiver(
@@ -66,12 +73,13 @@ impl GeyserPluginService {
             Receiver<GeyserPluginManagerRequest>,
             Arc<AtomicBool>,
         )>,
+        inclusions: Arc<RwLock<ProgramDatumInclusions>>,
     ) -> Result<Self, GeyserPluginServiceError> {
         info!(
             "Starting GeyserPluginService from config files: {:?}",
             geyser_plugin_config_files
         );
-        let mut plugin_manager = GeyserPluginManager::new();
+        let mut plugin_manager = GeyserPluginManager::new(inclusions);
 
         for geyser_plugin_config_file in geyser_plugin_config_files {
             Self::load_plugin(&mut plugin_manager, geyser_plugin_config_file)?;

diff --git a/ledger-tool/src/ledger_utils.rs b/ledger-tool/src/ledger_utils.rs
@@ -271,9 +271,16 @@ pub fn load_and_process_ledger(
 
         let (confirmed_bank_sender, confirmed_bank_receiver) = unbounded();
         drop(confirmed_bank_sender);
-        let geyser_service =
-            GeyserPluginService::new(confirmed_bank_receiver, &geyser_config_files)
-                .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?;
+        let inclusions_copy = process_options
+            .runtime_config
+            .program_datum_inclusions
+            .clone();
+        let geyser_service = GeyserPluginService::new(
+            confirmed_bank_receiver,
+            &geyser_config_files,
+            inclusions_copy,
+        )
+        .map_err(LoadAndProcessLedgerError::GeyserServiceSetup)?;
         (
             geyser_service.get_accounts_update_notifier(),
             geyser_service.get_transaction_notifier(),

diff --git a/ledger/src/blockstore_processor.rs b/ledger/src/blockstore_processor.rs
@@ -36,7 +36,7 @@ use {
     solana_rayon_threadlimit::{get_max_thread_count, get_thread_count},
     solana_runtime::{
         accounts_background_service::{AbsRequestSender, SnapshotRequestKind},
-        bank::{Bank, TransactionBalancesSet},
+        bank::{Bank, TransactionBalancesSet, TransactionDatumSet, TransactionOwnersSet},
         bank_forks::{BankForks, SetRootError},
         bank_utils,
         commitment::VOTE_THRESHOLD_SIZE,
@@ -167,7 +167,7 @@ pub fn execute_batch(
         vec![]
     };
 
-    let (tx_results, balances) = batch.bank().load_execute_and_commit_transactions(
+    let (tx_results, balances, datum, owners) = batch.bank().load_execute_and_commit_transactions(
         batch,
         MAX_PROCESSING_AGE,
         transaction_status_sender.is_some(),
@@ -233,6 +233,8 @@ pub fn execute_batch(
             transactions,
             execution_results,
             balances,
+            owners,
+            datum,
             token_balances,
             rent_debits,
             transaction_indexes.to_vec(),
@@ -2142,6 +2144,8 @@ pub struct TransactionStatusBatch {
     pub transactions: Vec<SanitizedTransaction>,
     pub execution_results: Vec<Option<TransactionExecutionDetails>>,
     pub balances: TransactionBalancesSet,
+    pub owners: TransactionOwnersSet,
+    pub datum: TransactionDatumSet,
     pub token_balances: TransactionTokenBalancesSet,
     pub rent_debits: Vec<RentDebits>,
     pub transaction_indexes: Vec<usize>,
@@ -2159,6 +2163,8 @@ impl TransactionStatusSender {
         transactions: Vec<SanitizedTransaction>,
         execution_results: Vec<TransactionExecutionResult>,
         balances: TransactionBalancesSet,
+        owners: TransactionOwnersSet,
+        datum: TransactionDatumSet,
         token_balances: TransactionTokenBalancesSet,
         rent_debits: Vec<RentDebits>,
         transaction_indexes: Vec<usize>,
@@ -2178,6 +2184,8 @@ impl TransactionStatusSender {
                     })
                     .collect(),
                 balances,
+                owners,
+                datum,
                 token_balances,
                 rent_debits,
                 transaction_indexes,
@@ -4279,6 +4287,8 @@ pub mod tests {
                 ..
             },
             _balances,
+            _datums,
+            _owners,
         ) = batch.bank().load_execute_and_commit_transactions(
             &batch,
             MAX_PROCESSING_AGE,

diff --git a/rpc-client/src/mock_sender.rs b/rpc-client/src/mock_sender.rs
@@ -193,6 +193,9 @@ impl RpcSender for MockSender {
                             fee: 0,
                             pre_balances: vec![499999999999999950, 50, 1],
                             post_balances: vec![499999999999999950, 50, 1],
+                            post_owners: None,
+                            pre_datum: None,
+                            post_datum: None,
                             inner_instructions: OptionSerializer::None,
                             log_messages: OptionSerializer::None,
                             pre_token_balances: OptionSerializer::None,
@@ -204,6 +207,8 @@ impl RpcSender for MockSender {
                         }),
                 },
                 block_time: Some(1628633791),
+                index_in_block: 0,
+                slot_second_idx: None,
             })?,
             "getTransactionCount" => json![1234],
             "getSlot" => json![0],