Adding config to support config and using env variable for tests and …

…value from file for others

bin/autobahn-router/src/ix_builder.rs

@@ -265,6 +265,7 @@ mod tests {
         async fn initialize(
             _rpc: &mut RouterRpcClient,
             _options: HashMap<String, String>,
+            _enable_compression: bool,
         ) -> anyhow::Result<Arc<dyn DexInterface>>
         where
             Self: Sized,

bin/autobahn-router/src/main.rs

@@ -235,6 +235,7 @@ async fn main() -> anyhow::Result<()> {
     );
     cropper.insert("program_name".to_string(), "Cropper".to_string());
 
+    let enable_compression = source_config.rpc_support_compression.unwrap_or_default();
     let mut router_rpc = RouterRpcClient {
         rpc: Box::new(RouterRpcWrapper {
             rpc: build_rpc(&source_config),
@@ -243,63 +244,89 @@ async fn main() -> anyhow::Result<()> {
 
     let dexs: Vec<Dex> = [
         dex::generic::build_dex!(
-            OrcaDex::initialize(&mut router_rpc, orca_config).await?,
+            OrcaDex::initialize(&mut router_rpc, orca_config, enable_compression).await?,
             &mango_data,
             config.orca.enabled,
             config.orca.add_mango_tokens,
             config.orca.take_all_mints,
             &config.orca.mints
         ),
         dex::generic::build_dex!(
-            OrcaDex::initialize(&mut router_rpc, cropper).await?,
+            OrcaDex::initialize(&mut router_rpc, cropper, enable_compression).await?,
             &mango_data,
             config.cropper.enabled,
             config.cropper.add_mango_tokens,
             config.cropper.take_all_mints,
             &config.cropper.mints
         ),
         dex::generic::build_dex!(
-            dex_saber::SaberDex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_saber::SaberDex::initialize(&mut router_rpc, HashMap::new(), enable_compression)
+                .await?,
             &mango_data,
             config.saber.enabled,
             config.saber.add_mango_tokens,
             config.saber.take_all_mints,
             &config.saber.mints
         ),
         dex::generic::build_dex!(
-            dex_raydium_cp::RaydiumCpDex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_raydium_cp::RaydiumCpDex::initialize(
+                &mut router_rpc,
+                HashMap::new(),
+                enable_compression
+            )
+            .await?,
             &mango_data,
             config.raydium_cp.enabled,
             config.raydium_cp.add_mango_tokens,
             config.raydium_cp.take_all_mints,
             &config.raydium_cp.mints
         ),
         dex::generic::build_dex!(
-            dex_raydium::RaydiumDex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_raydium::RaydiumDex::initialize(
+                &mut router_rpc,
+                HashMap::new(),
+                enable_compression
+            )
+            .await?,
             &mango_data,
             config.raydium.enabled,
             config.raydium.add_mango_tokens,
             config.raydium.take_all_mints,
             &config.raydium.mints
         ),
         dex::generic::build_dex!(
-            dex_openbook_v2::OpenbookV2Dex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_openbook_v2::OpenbookV2Dex::initialize(
+                &mut router_rpc,
+                HashMap::new(),
+                enable_compression
+            )
+            .await?,
             &mango_data,
             config.openbook_v2.enabled,
             config.openbook_v2.add_mango_tokens,
             config.openbook_v2.take_all_mints,
             &config.openbook_v2.mints
         ),
         dex::generic::build_dex!(
-            dex_infinity::InfinityDex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_infinity::InfinityDex::initialize(
+                &mut router_rpc,
+                HashMap::new(),
+                enable_compression
+            )
+            .await?,
             &mango_data,
             config.infinity.enabled,
             false,
             true,
             &vec![]
         ),
         dex::generic::build_dex!(
-            dex_invariant::InvariantDex::initialize(&mut router_rpc, HashMap::new()).await?,
+            dex_invariant::InvariantDex::initialize(
+                &mut router_rpc,
+                HashMap::new(),
+                enable_compression
+            )
+            .await?,
             &mango_data,
             config.invariant.enabled,
             config.invariant.take_all_mints,

bin/autobahn-router/src/mock.rs

@@ -56,6 +56,7 @@ pub mod test {
         async fn initialize(
             _rpc: &mut RouterRpcClient,
             _options: HashMap<String, String>,
+            _enable_compression: bool,
         ) -> anyhow::Result<Arc<dyn DexInterface>>
         where
             Self: Sized,

bin/autobahn-router/src/tests/dex_test_utils.rs

@@ -6,6 +6,7 @@ use std::sync::Arc;
 
 pub async fn get_all_dex(
     mut rpc_client: &mut RouterRpcClient,
+    enable_compression: bool,
 ) -> anyhow::Result<Vec<Arc<dyn DexInterface>>> {
     let orca_config = HashMap::from([
         (
@@ -23,13 +24,26 @@ pub async fn get_all_dex(
     ]);
 
     let dexs = [
-        dex_orca::OrcaDex::initialize(&mut rpc_client, orca_config).await?,
-        dex_orca::OrcaDex::initialize(&mut rpc_client, cropper_config).await?,
-        dex_saber::SaberDex::initialize(&mut rpc_client, HashMap::new()).await?,
-        dex_raydium_cp::RaydiumCpDex::initialize(&mut rpc_client, HashMap::new()).await?,
-        dex_raydium::RaydiumDex::initialize(&mut rpc_client, HashMap::new()).await?,
-        dex_openbook_v2::OpenbookV2Dex::initialize(&mut rpc_client, HashMap::new()).await?,
-        dex_infinity::InfinityDex::initialize(&mut rpc_client, HashMap::new()).await?,
+        dex_orca::OrcaDex::initialize(&mut rpc_client, orca_config, enable_compression).await?,
+        dex_orca::OrcaDex::initialize(&mut rpc_client, cropper_config, enable_compression).await?,
+        dex_saber::SaberDex::initialize(&mut rpc_client, HashMap::new(), enable_compression)
+            .await?,
+        dex_raydium_cp::RaydiumCpDex::initialize(
+            &mut rpc_client,
+            HashMap::new(),
+            enable_compression,
+        )
+        .await?,
+        dex_raydium::RaydiumDex::initialize(&mut rpc_client, HashMap::new(), enable_compression)
+            .await?,
+        dex_openbook_v2::OpenbookV2Dex::initialize(
+            &mut rpc_client,
+            HashMap::new(),
+            enable_compression,
+        )
+        .await?,
+        dex_infinity::InfinityDex::initialize(&mut rpc_client, HashMap::new(), enable_compression)
+            .await?,
     ];
 
     Ok(dexs.into_iter().collect())

bin/autobahn-router/src/tests/dump_all_dex.rs

@@ -24,7 +24,11 @@ mod tests {
         let rpc_url = env::var("RPC_HTTP_URL")?;
         let (mut rpc_client, _chain_data) = rpc::rpc_dumper_client(rpc_url, "all.lz4");
 
-        let dexs = dex_test_utils::get_all_dex(&mut rpc_client).await?;
+        let disable_compressed = std::env::var::<String>("DISABLE_COMRPESSED_GPA".to_string())
+            .unwrap_or("false".to_string());
+        let disable_compressed: bool = disable_compressed.trim().parse().unwrap();
+
+        let dexs = dex_test_utils::get_all_dex(&mut rpc_client, !disable_compressed).await?;
 
         for dex in &dexs {
             rpc::load_subscriptions(&mut rpc_client, dex.clone()).await?;

bin/autobahn-router/src/tests/performance_tests.rs

@@ -36,9 +36,13 @@ mod tests {
         let _vsol = Pubkey::from_str("vSoLxydx6akxyMD9XEcPvGYNGq6Nn66oqVb3UkGkei7").unwrap();
         let mnde = Pubkey::from_str("MNDEFzGvMt87ueuHvVU9VcTqsAP5b3fTGPsHuuPA5ey").unwrap();
 
+        let disable_compressed = std::env::var::<String>("DISABLE_COMRPESSED_GPA".to_string())
+            .unwrap_or("false".to_string());
+        let disable_compressed: bool = disable_compressed.trim().parse().unwrap();
+
         let (mut rpc_client, chain_data) = rpc::rpc_replayer_client("all.lz4");
         let chain_data = Arc::new(ChainDataAccountProvider::new(chain_data)) as AccountProviderView;
-        let dex_sources = dex_test_utils::get_all_dex(&mut rpc_client).await?;
+        let dex_sources = dex_test_utils::get_all_dex(&mut rpc_client, !disable_compressed).await?;
         let mut dexs = vec![];
         for dex in dex_sources {
             dexs.push(

bin/autobahn-router/src/tests/warmup_performance_tests.rs

@@ -24,9 +24,13 @@ mod tests {
 
         syscallstubs::deactivate_program_logs();
 
+        let disable_compressed = std::env::var::<String>("DISABLE_COMRPESSED_GPA".to_string())
+            .unwrap_or("false".to_string());
+        let disable_compressed: bool = disable_compressed.trim().parse().unwrap();
+
         let (mut rpc_client, chain_data) = rpc::rpc_replayer_client("all.lz4");
         let chain_data = Arc::new(ChainDataAccountProvider::new(chain_data)) as AccountProviderView;
-        let dex_sources = dex_test_utils::get_all_dex(&mut rpc_client).await?;
+        let dex_sources = dex_test_utils::get_all_dex(&mut rpc_client, !disable_compressed).await?;
         let mut dexs = vec![];
         for dex in dex_sources {
             dexs.push(

lib/dex-infinity/src/infinity.rs

@@ -36,6 +36,7 @@ impl DexInterface for InfinityDex {
     async fn initialize(
         rpc: &mut RouterRpcClient,
         _options: HashMap<String, String>,
+        _enable_compression: bool,
     ) -> anyhow::Result<Arc<dyn DexInterface>>
     where
         Self: Sized,

lib/dex-infinity/tests/test_infinity.rs

@@ -8,32 +8,38 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 
 #[tokio::test]
 async fn test_dump_input_data_infinity() -> anyhow::Result<()> {
+    let disable_compressed = std::env::var::<String>("DISABLE_COMRPESSED_GPA".to_string())
+        .unwrap_or("false".to_string());
+    let disable_compressed: bool = disable_compressed.trim().parse().unwrap();
+
     if router_test_lib::config_should_dump_mainnet_data() {
-        step_1_infinity().await?;
+        step_1_infinity(!disable_compressed).await?;
     }
 
-    step_2_infinity().await?;
+    step_2_infinity(!disable_compressed).await?;
 
     Ok(())
 }
 
-async fn step_1_infinity() -> anyhow::Result<()> {
+async fn step_1_infinity(enable_compression: bool) -> anyhow::Result<()> {
     let rpc_url = env::var("RPC_HTTP_URL")?;
     let (mut rpc_client, chain_data) = rpc::rpc_dumper_client(rpc_url, "infinity_dump.lz4");
 
     let options = HashMap::from([]);
-    let dex = dex_infinity::InfinityDex::initialize(&mut rpc_client, options).await?;
+    let dex =
+        dex_infinity::InfinityDex::initialize(&mut rpc_client, options, enable_compression).await?;
 
     generate_dex_rpc_dump::run_dump_mainnet_data(dex, rpc_client, chain_data).await?;
 
     Ok(())
 }
 
-async fn step_2_infinity() -> anyhow::Result<()> {
+async fn step_2_infinity(enable_compression: bool) -> anyhow::Result<()> {
     let (mut rpc_client, chain_data) = rpc::rpc_replayer_client("infinity_dump.lz4");
 
     let options = HashMap::from([]);
-    let dex = dex_infinity::InfinityDex::initialize(&mut rpc_client, options).await?;
+    let dex =
+        dex_infinity::InfinityDex::initialize(&mut rpc_client, options, enable_compression).await?;
 
     generate_dex_rpc_dump::run_dump_swap_ix("infinity_swap.lz4", dex, chain_data).await?;
 

lib/dex-invariant/src/invariant_dex.rs

@@ -231,11 +231,12 @@ impl DexInterface for InvariantDex {
     async fn initialize(
         rpc: &mut RouterRpcClient,
         _options: HashMap<String, String>,
+        enable_compression: bool,
     ) -> anyhow::Result<Arc<dyn DexInterface>>
     where
         Self: Sized,
     {
-        let mut pools = fetch_invariant_accounts(rpc, crate::id()).await?;
+        let mut pools = fetch_invariant_accounts(rpc, crate::id(), enable_compression).await?;
 
         let reserves = pools
             .iter()
@@ -460,6 +461,7 @@ impl DexInterface for InvariantDex {
 async fn fetch_invariant_accounts(
     rpc: &mut RouterRpcClient,
     program_id: Pubkey,
+    compression_enabled: bool,
 ) -> anyhow::Result<Vec<(Pubkey, Pool)>> {
     let config = RpcProgramAccountsConfig {
         filters: Some(vec![RpcFilterType::DataSize(Pool::LEN as u64)]),
@@ -471,7 +473,7 @@ async fn fetch_invariant_accounts(
     };
 
     let snapshot = rpc
-        .get_program_accounts_with_config(&program_id, config)
+        .get_program_accounts_with_config(&program_id, config, compression_enabled)
         .await?;
 
     let result = snapshot

lib/dex-invariant/tests/test_invariant.rs

@@ -8,31 +8,50 @@ use router_lib::test_tools::{generate_dex_rpc_dump, rpc};
 #[tokio::test]
 async fn test_dump_input_data_invariant() -> anyhow::Result<()> {
     let options = HashMap::from([]);
+    let disable_compressed = std::env::var::<String>("DISABLE_COMRPESSED_GPA".to_string())
+        .unwrap_or("false".to_string());
+    let disable_compressed: bool = disable_compressed.trim().parse().unwrap();
 
     if router_test_lib::config_should_dump_mainnet_data() {
-        invariant_step_1(&options).await?;
+        invariant_step_1(&options, !disable_compressed).await?;
     }
 
-    invariant_step_2(&options).await?;
+    invariant_step_2(&options, !disable_compressed).await?;
 
     Ok(())
 }
 
-async fn invariant_step_1(options: &HashMap<String, String>) -> anyhow::Result<()> {
+async fn invariant_step_1(
+    options: &HashMap<String, String>,
+    enable_compression: bool,
+) -> anyhow::Result<()> {
     let rpc_url = env::var("RPC_HTTP_URL")?;
 
     let (mut rpc_client, chain_data) = rpc::rpc_dumper_client(rpc_url, "invariant_swap.lz4");
-    let dex = dex_invariant::InvariantDex::initialize(&mut rpc_client, options.clone()).await?;
+    let dex = dex_invariant::InvariantDex::initialize(
+        &mut rpc_client,
+        options.clone(),
+        enable_compression,
+    )
+    .await?;
 
     generate_dex_rpc_dump::run_dump_mainnet_data(dex, rpc_client, chain_data).await?;
 
     Ok(())
 }
 
-async fn invariant_step_2(options: &HashMap<String, String>) -> anyhow::Result<()> {
+async fn invariant_step_2(
+    options: &HashMap<String, String>,
+    enable_compression: bool,
+) -> anyhow::Result<()> {
     let (mut rpc_client, chain_data) = rpc::rpc_replayer_client("invariant_swap.lz4");
 
-    let dex = dex_invariant::InvariantDex::initialize(&mut rpc_client, options.clone()).await?;
+    let dex = dex_invariant::InvariantDex::initialize(
+        &mut rpc_client,
+        options.clone(),
+        enable_compression,
+    )
+    .await?;
 
     generate_dex_rpc_dump::run_dump_swap_ix("invariant_swap.lz4", dex, chain_data).await?;
 

lib/dex-openbook-v2/src/openbook_v2_dex.rs

@@ -32,11 +32,12 @@ impl DexInterface for OpenbookV2Dex {
     async fn initialize(
         rpc: &mut RouterRpcClient,
         _options: HashMap<String, String>,
+        enable_compression: bool,
     ) -> anyhow::Result<Arc<dyn DexInterface>>
     where
         Self: Sized,
     {
-        let markets = fetch_openbook_v2_account(rpc, openbook_v2::id())
+        let markets = fetch_openbook_v2_account(rpc, openbook_v2::id(), enable_compression)
             .await?
             .into_iter()
             .filter(|x| x.1.open_orders_admin.is_none())
@@ -404,6 +405,7 @@ impl DexInterface for OpenbookV2Dex {
 async fn fetch_openbook_v2_account(
     rpc: &mut RouterRpcClient,
     program_id: Pubkey,
+    enable_compression: bool,
 ) -> anyhow::Result<Vec<(Pubkey, Market)>> {
     let config = RpcProgramAccountsConfig {
         filters: Some(vec![
@@ -422,7 +424,7 @@ async fn fetch_openbook_v2_account(
     };
 
     let snapshot = rpc
-        .get_program_accounts_with_config(&program_id, config) // todo use compression here
+        .get_program_accounts_with_config(&program_id, config, enable_compression) // todo use compression here
         .await?;
 
     let result = snapshot

lib/dex-openbook-v2/src/openbook_v2_ix_builder.rs

@@ -9,7 +9,6 @@ use solana_program::pubkey::Pubkey;
 use solana_sdk::clock::Clock;
 use solana_sdk::instruction::AccountMeta;
 use solana_sdk::sysvar::SysvarId;
-use std::str::FromStr;
 
 pub const INCLUDED_MAKERS_COUNT: usize = 2;