From fa96d4267addfda550f8e1a777a391d4c85d4a1e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 7 Sep 2023 16:24:17 +0200 Subject: [PATCH 01/42] make client an ipv6 endpoint --- services/src/tpu_utils/quic_proxy_connection_manager.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index e2923097..9d7daa38 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::net::{IpAddr, Ipv6Addr, SocketAddr}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; @@ -112,7 +112,7 @@ impl QuicProxyConnectionManager { let mut endpoint = { let client_socket = - solana_net_utils::bind_in_range(IpAddr::V4(Ipv4Addr::UNSPECIFIED), (8000, 10000)) + solana_net_utils::bind_in_range(IpAddr::V6(Ipv6Addr::UNSPECIFIED), (8000, 10000)) .expect("create_endpoint bind_in_range") .1; let config = EndpointConfig::default(); From 440ab7f418b40180d28700edc35c178c79e86136 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 7 Sep 2023 16:27:47 +0200 Subject: [PATCH 02/42] use simplifind client binding --- services/src/tpu_utils/quic_proxy_connection_manager.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 9d7daa38..0d9adf03 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{IpAddr, Ipv6Addr, SocketAddr}; +use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; @@ -111,10 +111,7 @@ impl QuicProxyConnectionManager { const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; let mut endpoint = { - let client_socket = - solana_net_utils::bind_in_range(IpAddr::V6(Ipv6Addr::UNSPECIFIED), (8000, 10000)) - .expect("create_endpoint bind_in_range") - .1; + let client_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); let config = EndpointConfig::default(); Endpoint::new(config, None, client_socket, TokioRuntime) .expect("create_endpoint quinn::Endpoint::new") From 208e8eb00c0fcb2050c3a33ba3d0de6601985598 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 7 Sep 2023 16:46:10 +0200 Subject: [PATCH 03/42] bind client socket dual-stack --- services/src/tpu_utils/quic_proxy_connection_manager.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 0d9adf03..37f294e0 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{IpAddr, Ipv6Addr, SocketAddr, UdpSocket}; +use std::net::{SocketAddr, UdpSocket}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; @@ -111,7 +111,8 @@ impl QuicProxyConnectionManager { const ALPN_TPU_FORWARDPROXY_PROTOCOL_ID: &[u8] = b"solana-tpu-forward-proxy"; let mut endpoint = { - let client_socket = UdpSocket::bind("127.0.0.1:0").unwrap(); + // Binding on :: will also listen on IPv4 (dual-stack). + let client_socket = UdpSocket::bind("[::]:0").unwrap(); let config = EndpointConfig::default(); Endpoint::new(config, None, client_socket, TokioRuntime) .expect("create_endpoint quinn::Endpoint::new") From 7b1ad673251ef9a48c53e4f35af36fab66a344c9 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 8 Sep 2023 12:07:37 +0200 Subject: [PATCH 04/42] WIP: send transactions to TPU via QUIC instead of RPC - TODO: add cli option to configure --- Cargo.lock | 2 + bench/Cargo.toml | 2 + bench/src/main.rs | 69 +++++++++++++++++++++++++++-------- quic-forward-proxy/src/lib.rs | 4 +- 4 files changed, 59 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e9a3c58..bbd02eaf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -574,6 +574,7 @@ name = "bench" version = "0.2.3" dependencies = [ "anyhow", + "bincode", "clap 4.4.2", "csv", "dashmap", @@ -585,6 +586,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde_json", + "solana-lite-rpc-quic-forward-proxy", "solana-rpc-client", "solana-sdk", "tokio", diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 3def8bf0..9c917d9f 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -4,6 +4,7 @@ version = "0.2.3" edition = "2021" [dependencies] +solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } solana-sdk = { workspace = true } solana-rpc-client = { workspace = true } log = { workspace = true } @@ -13,6 +14,7 @@ serde_json = { workspace = true } clap = { workspace = true } tokio = { version = "1.28.2", features = ["full", "fs"]} tracing-subscriber = { workspace = true } +bincode = { workspace = true } csv = "1.2.1" dirs = "5.0.0" rand = "0.8.5" diff --git a/bench/src/main.rs b/bench/src/main.rs index 3b278bda..11d23ef0 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -16,10 +16,17 @@ use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, }; +use std::sync::atomic::AtomicBool; +use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::Transaction; use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; +use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; +use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; +use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] async fn main() { @@ -155,7 +162,20 @@ async fn bench( tx_metric_sx: UnboundedSender, log_txs: bool, ) -> Metric { - let map_of_txs = Arc::new(DashMap::new()); + let map_of_txs: Arc> = Arc::new(DashMap::new()); + let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); + + { + let validator_identity = ValidatorIdentity::new(None); + let exit_signal = Arc::new(AtomicBool::new(false)); + let _jh = tokio::spawn(tx_forwarder( + validator_identity, + forward_receiver, + exit_signal, + + )); + } + // transaction sender task { let map_of_txs = map_of_txs.clone(); @@ -169,21 +189,38 @@ async fn bench( let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); let start_time = Instant::now(); - match rpc_client.send_transaction(&tx).await { - Ok(signature) => { - map_of_txs.insert( - signature, - TxSendData { - sent_duration: start_time.elapsed(), - sent_instant: Instant::now(), - sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - }, - ); - } - Err(e) => { - warn!("tx send failed with error {}", e); - } - } + // match rpc_client.send_transaction(&tx).await { + // Ok(signature) => { + // map_of_txs.insert( + // signature, + // TxSendData { + // sent_duration: start_time.elapsed(), + // sent_instant: Instant::now(), + // sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + // }, + // ); + // } + // Err(e) => { + // warn!("tx send failed with error {}", e); + // } + // } + let tpu_address = "127.0.0.1:1033".parse().unwrap(); + + let tx_raw = bincode::serialize::(&tx).unwrap(); + let packet = ForwardPacket::new( + vec![tx_raw], + tpu_address, + 424242, + ); + + forwarder_channel.send(packet).await; + + map_of_txs.insert(tx.get_signature().clone(), TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }); + } }); } diff --git a/quic-forward-proxy/src/lib.rs b/quic-forward-proxy/src/lib.rs index ffc51a52..34921817 100644 --- a/quic-forward-proxy/src/lib.rs +++ b/quic-forward-proxy/src/lib.rs @@ -2,12 +2,12 @@ mod cli; mod inbound; -mod outbound; +pub mod outbound; pub mod proxy; pub mod proxy_request_format; mod quic_util; mod quinn_auto_reconnect; -mod shared; +pub mod shared; pub mod tls_config_provider_client; pub mod tls_config_provider_server; pub mod tls_self_signed_pair_generator; From c87d64c38d4de2a48fd978d5c3fee67093f4ee19 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 11:52:31 +0200 Subject: [PATCH 05/42] cheap hack putting leaders in a local file --- bench/src/main.rs | 52 ++++++++++++++++++++------- cd/lite-rpc.toml | 18 ++++++++++ cd/quic-forward-proxy.toml | 13 +++++++ services/src/tpu_utils/tpu_service.rs | 24 ++++++++++++- 4 files changed, 93 insertions(+), 14 deletions(-) create mode 100644 cd/lite-rpc.toml create mode 100644 cd/quic-forward-proxy.toml diff --git a/bench/src/main.rs b/bench/src/main.rs index 11d23ef0..5dcbfc65 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -1,3 +1,6 @@ +use std::fs::read_to_string; +use std::net::{SocketAddr, SocketAddrV4}; +use std::str::FromStr; use bench::{ cli::Args, helpers::BenchHelper, @@ -24,6 +27,8 @@ use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; +use solana_lite_rpc_cluster_endpoints::json_rpc_leaders_getter::JsonRpcLeaderGetter; +use solana_lite_rpc_core::leaders_fetcher_trait::LeaderFetcherInterface; use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; @@ -188,6 +193,9 @@ async fn bench( for rand_string in rand_strings { let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); + + let leader_addrs = read_leaders_from_file("/Users/stefan/mango/code/lite-rpc/leaders.dat").expect("leaders.dat file"); + let start_time = Instant::now(); // match rpc_client.send_transaction(&tx).await { // Ok(signature) => { @@ -204,22 +212,26 @@ async fn bench( // warn!("tx send failed with error {}", e); // } // } - let tpu_address = "127.0.0.1:1033".parse().unwrap(); + // let tpu_address = "127.0.0.1:1033".parse().unwrap(); - let tx_raw = bincode::serialize::(&tx).unwrap(); - let packet = ForwardPacket::new( - vec![tx_raw], - tpu_address, - 424242, - ); + for tpu_address in &leader_addrs { + let tx_raw = bincode::serialize::(&tx).unwrap(); + let packet = ForwardPacket::new( + vec![tx_raw], + SocketAddr::from(*tpu_address), + 424242, + ); - forwarder_channel.send(packet).await; + forwarder_channel.send(packet).await; - map_of_txs.insert(tx.get_signature().clone(), TxSendData { - sent_duration: start_time.elapsed(), - sent_instant: Instant::now(), - sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - }); + println!("sent tx {}", tx.get_signature()); + + map_of_txs.insert(tx.get_signature().clone(), TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }); + } } }); @@ -268,3 +280,17 @@ async fn bench( metric.finalize(); metric } + + +fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { + let leader_file = read_to_string(leaders_file)?; + let mut leader_addrs = vec![]; + for line in leader_file.lines() { + println!("line: {}", line); + let socket_addr = SocketAddrV4::from_str(line).unwrap(); + println!("socket_addr: {}", socket_addr); + leader_addrs.push(socket_addr); + } + Ok(leader_addrs) +} + diff --git a/cd/lite-rpc.toml b/cd/lite-rpc.toml new file mode 100644 index 00000000..f6b45937 --- /dev/null +++ b/cd/lite-rpc.toml @@ -0,0 +1,18 @@ +app = "solana-lite-rpc" +kill_signal = "SIGINT" +kill_timeout = 5 + +[build] + dockerfile = "../Dockerfile" + +[experimental] + cmd = ["sh", "-c", "lite-rpc --rpc-addr $RPC_URL --ws-addr $WS_URL --quic-proxy-addr $QUIC_PROXY_ADDR"] + +[env] + PORT_HTTP = "8890" + PORT_WS = "8891" + RUST_LOG = "info" + + [metrics] + path = "/metrics" + port = 9091 diff --git a/cd/quic-forward-proxy.toml b/cd/quic-forward-proxy.toml new file mode 100644 index 00000000..edefe136 --- /dev/null +++ b/cd/quic-forward-proxy.toml @@ -0,0 +1,13 @@ +app = "solana-quic-forward-proxy" +kill_signal = "SIGINT" +kill_timeout = 5 + +[build] + dockerfile = "../Dockerfile" + +[experimental] + cmd = ["solana-lite-rpc-quic-forward-proxy"] + +[env] + PROXY_LISTEN_ADDR = "[::]:11111" + RUST_LOG = "debug" diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index c093ef51..804d39fb 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -15,6 +15,12 @@ use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, }; +use std::collections::HashMap; +use std::fs::{File, read_to_string}; +use std::io::Write; +use std::net::SocketAddr; +use itertools::Itertools; +use solana_sdk::pubkey::Pubkey; use tokio::time::Duration; #[derive(Clone, Copy)] @@ -120,7 +126,7 @@ impl TpuService { .get_slot_leaders(load_slot, last_slot) .await?; // get next leader with its tpu port - let connections_to_keep = next_leaders + let connections_to_keep: HashMap = next_leaders .iter() .map(|x| { let contact_info = cluster_nodes.get(&x.pubkey); @@ -139,6 +145,11 @@ impl TpuService { }) .collect(); + + dump_leaders_to_file(connections_to_keep.values().collect_vec()); + // read_file(); + + match &self.connection_manager { DirectTpu { tpu_connection_manager, @@ -187,3 +198,14 @@ impl TpuService { }) } } + +fn dump_leaders_to_file(leaders: Vec<&SocketAddr>) { + // will create/truncate file + // 69.197.20.37:8009 + let mut out_file = File::create("leaders.dat").unwrap(); + + for leader_addr in &leaders { + write!(out_file, "{}\n", leader_addr).unwrap(); + } +} + From a148ed5d7aeaf90cae69c50c3c1ea83c71fc6d17 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 11:53:51 +0200 Subject: [PATCH 06/42] compile --- bench/src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index 5dcbfc65..a13172f6 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -27,8 +27,6 @@ use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; -use solana_lite_rpc_cluster_endpoints::json_rpc_leaders_getter::JsonRpcLeaderGetter; -use solana_lite_rpc_core::leaders_fetcher_trait::LeaderFetcherInterface; use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; From 466910b70cc3572df6482da417c219cb84a9adb1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 11:58:03 +0200 Subject: [PATCH 07/42] fix leaders.dat path --- bench/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index a13172f6..d766b0c6 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -192,7 +192,7 @@ async fn bench( let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); - let leader_addrs = read_leaders_from_file("/Users/stefan/mango/code/lite-rpc/leaders.dat").expect("leaders.dat file"); + let leader_addrs = read_leaders_from_file("leaders.dat").expect("leaders.dat file"); let start_time = Instant::now(); // match rpc_client.send_transaction(&tx).await { From 5c627eccc98b340ab52f2225a3eca5e8f83dcea5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 12:00:03 +0200 Subject: [PATCH 08/42] reduce log output --- bench/src/main.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index d766b0c6..cab4bcd5 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -212,6 +212,7 @@ async fn bench( // } // let tpu_address = "127.0.0.1:1033".parse().unwrap(); + println!("sent tx {} to {} tpu nodes", tx.get_signature(), leader_addrs.len()); for tpu_address in &leader_addrs { let tx_raw = bincode::serialize::(&tx).unwrap(); let packet = ForwardPacket::new( @@ -222,8 +223,6 @@ async fn bench( forwarder_channel.send(packet).await; - println!("sent tx {}", tx.get_signature()); - map_of_txs.insert(tx.get_signature().clone(), TxSendData { sent_duration: start_time.elapsed(), sent_instant: Instant::now(), @@ -284,9 +283,7 @@ fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result Date: Fri, 15 Sep 2023 12:04:34 +0200 Subject: [PATCH 09/42] soft handling of transaction_channel closed --- quic-forward-proxy/src/outbound/tx_forward.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 8aeab522..fb8034ca 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -63,7 +63,7 @@ pub async fn tx_forwarder( transaction_channel .recv() .await - .expect("channel closed unexpectedly"), + .ok_or(anyhow::anyhow!("transaction_channel closed"))? ); let tpu_address = forward_packet.tpu_address; From 1f9e46428d3773f1707f70f553258ee299f7a30e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 12:12:49 +0200 Subject: [PATCH 10/42] deal with log errors --- bench/src/main.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index cab4bcd5..8bc1c837 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -9,7 +9,7 @@ use bench::{ use clap::Parser; use dashmap::DashMap; use futures::future::join_all; -use log::{error, info, warn}; +use log::{debug, error, info, warn}; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, @@ -20,6 +20,7 @@ use std::sync::{ Arc, }; use std::sync::atomic::AtomicBool; +use anyhow::Context; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::signature::Signature; use solana_sdk::transaction::Transaction; @@ -27,6 +28,7 @@ use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; +use tracing_subscriber::fmt::format; use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; @@ -212,7 +214,7 @@ async fn bench( // } // let tpu_address = "127.0.0.1:1033".parse().unwrap(); - println!("sent tx {} to {} tpu nodes", tx.get_signature(), leader_addrs.len()); + debug!("sent tx {} to {} tpu nodes", tx.get_signature(), leader_addrs.len()); for tpu_address in &leader_addrs { let tx_raw = bincode::serialize::(&tx).unwrap(); let packet = ForwardPacket::new( @@ -283,7 +285,7 @@ fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result Date: Fri, 15 Sep 2023 12:20:06 +0200 Subject: [PATCH 11/42] leaders.dat atomic write --- services/src/tpu_utils/tpu_service.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 804d39fb..0eb42f04 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -202,10 +202,12 @@ impl TpuService { fn dump_leaders_to_file(leaders: Vec<&SocketAddr>) { // will create/truncate file // 69.197.20.37:8009 - let mut out_file = File::create("leaders.dat").unwrap(); - + let mut out_file = File::create("leaders.dat.tmp").unwrap(); for leader_addr in &leaders { write!(out_file, "{}\n", leader_addr).unwrap(); } + out_file.flush().unwrap(); + + std::fs::rename("leaders.dat.tmp", "leaders.dat").unwrap(); } From 529548ee612cf8b4b2725d2a41fb95346e61dc89 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 12:26:25 +0200 Subject: [PATCH 12/42] check for leader.dat age --- bench/src/main.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index 8bc1c837..7a6f272f 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -1,4 +1,5 @@ -use std::fs::read_to_string; +use std::fs; +use std::fs::{File, read_to_string}; use std::net::{SocketAddr, SocketAddrV4}; use std::str::FromStr; use bench::{ @@ -20,6 +21,7 @@ use std::sync::{ Arc, }; use std::sync::atomic::AtomicBool; +use std::time::SystemTime; use anyhow::Context; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::signature::Signature; @@ -282,6 +284,9 @@ async fn bench( fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { + let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); + let file_age = SystemTime::now().duration_since(last_modified).unwrap(); + assert!(file_age.as_millis() < 1000, "leaders.dat is outdated ({:?})", file_age); let leader_file = read_to_string(leaders_file)?; let mut leader_addrs = vec![]; for line in leader_file.lines() { From 6d8acf0517dc62a5a01e264502e25bb2390661e9 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 12:34:03 +0200 Subject: [PATCH 13/42] temp diable channel drain --- quic-forward-proxy/src/outbound/tx_forward.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index fb8034ca..2f81c0e8 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -140,15 +140,15 @@ pub async fn tx_forwarder( let mut transactions_batch: Vec> = packet.transactions.clone(); - 'more: while let Ok(more) = per_connection_receiver.try_recv() { - if more.tpu_address != tpu_address { - continue 'more; - } - if !sharder.matching(more.shard_hash) { - continue 'more; - } - transactions_batch.extend(more.transactions.clone()); - } + // 'more: while let Ok(more) = per_connection_receiver.try_recv() { + // if more.tpu_address != tpu_address { + // continue 'more; + // } + // if !sharder.matching(more.shard_hash) { + // continue 'more; + // } + // transactions_batch.extend(more.transactions.clone()); + // } debug!( "forwarding transaction batch of size {} to address {}", From 0a2df9a18e86ccff87db4175ab67a830084ceaba Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 12:35:05 +0200 Subject: [PATCH 14/42] re-enable diable channel drain --- quic-forward-proxy/src/outbound/tx_forward.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 2f81c0e8..fb8034ca 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -140,15 +140,15 @@ pub async fn tx_forwarder( let mut transactions_batch: Vec> = packet.transactions.clone(); - // 'more: while let Ok(more) = per_connection_receiver.try_recv() { - // if more.tpu_address != tpu_address { - // continue 'more; - // } - // if !sharder.matching(more.shard_hash) { - // continue 'more; - // } - // transactions_batch.extend(more.transactions.clone()); - // } + 'more: while let Ok(more) = per_connection_receiver.try_recv() { + if more.tpu_address != tpu_address { + continue 'more; + } + if !sharder.matching(more.shard_hash) { + continue 'more; + } + transactions_batch.extend(more.transactions.clone()); + } debug!( "forwarding transaction batch of size {} to address {}", From 70617c70d027b2a5f7ac0a559bfdd12a1e64dc6b Mon Sep 17 00:00:00 2001 From: Godmode Galactus Date: Fri, 15 Sep 2023 15:24:53 +0200 Subject: [PATCH 15/42] merge gm patch --- Cargo.lock | 4 +- bench/Cargo.toml | 4 + bench/src/main.rs | 93 +++++++++++++------ .../src/tpu_utils/tpu_connection_manager.rs | 2 +- 4 files changed, 74 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bbd02eaf..4bde7b11 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,9 +586,11 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde_json", - "solana-lite-rpc-quic-forward-proxy", + "solana-lite-rpc-core", + "solana-lite-rpc-services", "solana-rpc-client", "solana-sdk", + "solana-streamer", "tokio", "tracing-subscriber", ] diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 9c917d9f..d8656611 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -22,4 +22,8 @@ rand_chacha = "0.3.1" futures = { workspace = true } dashmap = { workspace = true } lazy_static = "1.4.0" +bincode = { workspace = true } +solana-lite-rpc-core = {workspace = true} +solana-lite-rpc-services = {workspace = true} +solana-streamer = {workspace = true} diff --git a/bench/src/main.rs b/bench/src/main.rs index 7a6f272f..c8dea4bc 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -10,16 +10,22 @@ use bench::{ use clap::Parser; use dashmap::DashMap; use futures::future::join_all; -use log::{debug, error, info, warn}; +use log::{debug, error, info}; +use solana_lite_rpc_core::{structures::identity_stakes::IdentityStakesData, stores::tx_store::TxStore, quic_connection_utils::QuicConnectionParameters}; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, - slot_history::Slot, + slot_history::Slot, pubkey::Pubkey, }; -use std::sync::{ +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::{sync::{ atomic::{AtomicU64, Ordering}, Arc, -}; +}, net::{IpAddr, Ipv4Addr}, collections::HashMap}; +use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::Transaction; use std::sync::atomic::AtomicBool; use std::time::SystemTime; use anyhow::Context; @@ -67,6 +73,35 @@ async fn main() { let slot = rpc_client.get_slot().await.unwrap(); let block_hash: Arc> = Arc::new(RwLock::new(bh)); let current_slot = Arc::new(AtomicU64::new(slot)); + + let (forwarder_channel, _) = tokio::sync::broadcast::channel(1000); + + let identity = Keypair::new(); + + let (certificate, key) = new_self_signed_tls_certificate( + &identity, + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ) + .expect("Failed to initialize QUIC client certificates"); + + let tpu_connection_manager = TpuConnectionManager::new(certificate, key, 4).await; + let mut connections_to_keep = HashMap::new(); + connections_to_keep.insert(Pubkey::new_unique(), std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033)); + let identity_stakes = IdentityStakesData::default(); + let tx_store = TxStore::default(); + let connection_parameters = QuicConnectionParameters { + write_timeout: Duration::from_millis(500), + connection_retry_count: 10, + connection_timeout: Duration::from_millis(1000), + finalize_timeout: Duration::from_millis(100), + max_number_of_connections: 8, + number_of_transactions_per_unistream: 1, + unistream_timeout: Duration::from_millis(500), + }; + + let forwarder_channel = Arc::new(forwarder_channel); + tpu_connection_manager.update_connections(forwarder_channel.clone(), connections_to_keep, identity_stakes, tx_store, connection_parameters).await; + { // block hash updater task let block_hash = block_hash.clone(); @@ -118,6 +153,7 @@ async fn main() { current_slot.clone(), tx_log_sx.clone(), log_transactions, + forwarder_channel.clone(), ))); // wait for an interval run_interval_ms.tick().await; @@ -168,6 +204,7 @@ async fn bench( current_slot: Arc, tx_metric_sx: UnboundedSender, log_txs: bool, + forwarder_channel : Arc)>> ) -> Metric { let map_of_txs: Arc> = Arc::new(DashMap::new()); let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); @@ -182,17 +219,16 @@ async fn bench( )); } - // transaction sender task { let map_of_txs = map_of_txs.clone(); - let rpc_client = rpc_client.clone(); let current_slot = current_slot.clone(); + let forwarder_channel = forwarder_channel.clone(); tokio::spawn(async move { let map_of_txs = map_of_txs.clone(); let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); - for rand_string in rand_strings { + let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); @@ -249,29 +285,32 @@ async fn bench( tokio::time::sleep(Duration::from_millis(1)).await; continue; } - - if let Ok(res) = rpc_client.get_signature_statuses(&signatures).await { - for (i, signature) in signatures.iter().enumerate() { - let tx_status = &res.value[i]; - if tx_status.is_some() { - let tx_data = map_of_txs.get(signature).unwrap(); - let time_to_confirm = tx_data.sent_instant.elapsed(); - metric.add_successful_transaction(tx_data.sent_duration, time_to_confirm); - - if log_txs { - let _ = tx_metric_sx.send(TxMetricData { - signature: signature.to_string(), - sent_slot: tx_data.sent_slot, - confirmed_slot: current_slot.load(Ordering::Relaxed), - time_to_send_in_millis: tx_data.sent_duration.as_millis() as u64, - time_to_confirm_in_millis: time_to_confirm.as_millis() as u64, - }); + let chunks = signatures.chunks(100).collect::>(); + for chunk in chunks { + if let Ok(res) = rpc_client.get_signature_statuses(&chunk).await { + for (i, signature) in chunk.iter().enumerate() { + let tx_status = &res.value[i]; + if tx_status.is_some() { + let tx_data = map_of_txs.get(signature).unwrap(); + let time_to_confirm = tx_data.sent_instant.elapsed(); + metric.add_successful_transaction(tx_data.sent_duration, time_to_confirm); + + if log_txs { + let _ = tx_metric_sx.send(TxMetricData { + signature: signature.to_string(), + sent_slot: tx_data.sent_slot, + confirmed_slot: current_slot.load(Ordering::Relaxed), + time_to_send_in_millis: tx_data.sent_duration.as_millis() as u64, + time_to_confirm_in_millis: time_to_confirm.as_millis() as u64, + }); + } + drop(tx_data); + map_of_txs.remove(signature); + confirmed_count += 1; } - drop(tx_data); - map_of_txs.remove(signature); - confirmed_count += 1; } } + tokio::time::sleep(Duration::from_millis(50)).await; } } diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index fc942004..59b9016a 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -119,7 +119,7 @@ impl ActiveConnection { "Broadcast channel error on recv for {} error {} - continue", identity, e ); - continue; + break; } }; From 7a77ae96fe899e13252c683e73744b4d29d3b739 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 15:44:41 +0200 Subject: [PATCH 16/42] merge gm patch --- bench/Cargo.toml | 1 - bench/src/main.rs | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/bench/Cargo.toml b/bench/Cargo.toml index d8656611..ceba95d9 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -22,7 +22,6 @@ rand_chacha = "0.3.1" futures = { workspace = true } dashmap = { workspace = true } lazy_static = "1.4.0" -bincode = { workspace = true } solana-lite-rpc-core = {workspace = true} solana-lite-rpc-services = {workspace = true} solana-streamer = {workspace = true} diff --git a/bench/src/main.rs b/bench/src/main.rs index c8dea4bc..baf446c5 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -11,7 +11,8 @@ use clap::Parser; use dashmap::DashMap; use futures::future::join_all; use log::{debug, error, info}; -use solana_lite_rpc_core::{structures::identity_stakes::IdentityStakesData, stores::tx_store::TxStore, quic_connection_utils::QuicConnectionParameters}; +use solana_lite_rpc_core::tx_store::TxStore; +use solana_lite_rpc_core::{structures::identity_stakes::IdentityStakesData, quic_connection_utils::QuicConnectionParameters}; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_sdk::{ @@ -29,9 +30,6 @@ use solana_sdk::transaction::Transaction; use std::sync::atomic::AtomicBool; use std::time::SystemTime; use anyhow::Context; -use solana_rpc_client::rpc_client::SerializableTransaction; -use solana_sdk::signature::Signature; -use solana_sdk::transaction::Transaction; use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, From ee100d43fca6519fc5b4b952bd8a94415bebf87b Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 19:46:39 +0200 Subject: [PATCH 17/42] add proxy to CI build --- Dockerfile | 4 +++- cd/quic-forward-proxy.toml | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 0678df6d..5c826556 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,10 +14,12 @@ FROM base as build COPY --from=plan /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release --bin lite-rpc +RUN cargo build --release --bin lite-rpc --bin solana-lite-rpc-quic-forward-proxy --bin bench FROM debian:bullseye-slim as run RUN apt-get update && apt-get -y install ca-certificates libc6 +COPY --from=build /app/target/release/solana-lite-rpc-quic-forward-proxy /usr/local/bin/ COPY --from=build /app/target/release/lite-rpc /usr/local/bin/ +COPY --from=build /app/target/release/bench /usr/local/bin/ CMD lite-rpc --rpc-addr "$RPC_URL" --ws-addr "$WS_URL" diff --git a/cd/quic-forward-proxy.toml b/cd/quic-forward-proxy.toml index edefe136..21d34d60 100644 --- a/cd/quic-forward-proxy.toml +++ b/cd/quic-forward-proxy.toml @@ -11,3 +11,10 @@ kill_timeout = 5 [env] PROXY_LISTEN_ADDR = "[::]:11111" RUST_LOG = "debug" + +[[services]] + internal_port = 11111 + protocol = "udp" + +[[services.ports]] + port = "11111" From 757ef0a1cd6f078351d4f9699440ad9bf7aac48d Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Fri, 15 Sep 2023 20:16:42 +0200 Subject: [PATCH 18/42] more log info on leaders.data --- bench/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index baf446c5..1a34db6d 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -323,7 +323,7 @@ async fn bench( fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); let file_age = SystemTime::now().duration_since(last_modified).unwrap(); - assert!(file_age.as_millis() < 1000, "leaders.dat is outdated ({:?})", file_age); + assert!(file_age.as_millis() < 1000, "leaders.dat is outdated ({:?}) - pls run patched lite-rpc service", file_age); let leader_file = read_to_string(leaders_file)?; let mut leader_addrs = vec![]; for line in leader_file.lines() { From e8f5a8c39463a0f4776180d2902416e77c086623 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 11:21:26 +0200 Subject: [PATCH 19/42] add simple tool to send transaction against proxy --- Cargo.toml | 4 +- core/src/proxy_request_format.rs | 2 +- .../Cargo.toml | 1 + .../src/proxy_request_format.rs | 2 +- .../quic_proxy_connection_manager.rs | 44 ++++++++++++++++++- 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d44ed090..9460ba5f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -50,8 +50,8 @@ prometheus = "0.13.3" lazy_static = "1.4.0" dotenv = "0.15.0" async-channel = "1.8.0" -quinn = "0.9.3" -rustls = { version = "=0.20.8", default-features = false } +quinn = "0.9.4" +rustls = { version = "0.20.9", default-features = false } solana-lite-rpc-services = {path = "services", version="0.2.3"} solana-lite-rpc-core = {path = "core", version="0.2.3"} solana-lite-rpc-cluster-endpoints = {path = "cluster-endpoints", version="0.2.3"} diff --git a/core/src/proxy_request_format.rs b/core/src/proxy_request_format.rs index f1f32c4d..100fe70c 100644 --- a/core/src/proxy_request_format.rs +++ b/core/src/proxy_request_format.rs @@ -40,7 +40,7 @@ impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "TpuForwardingRequest t9 {} tpu nodes", + "TpuForwardingRequest to {} tpu nodes", &self.tpu_nodes.len(), ) } diff --git a/quic-forward-proxy-integration-test/Cargo.toml b/quic-forward-proxy-integration-test/Cargo.toml index d2481d2a..8de47263 100644 --- a/quic-forward-proxy-integration-test/Cargo.toml +++ b/quic-forward-proxy-integration-test/Cargo.toml @@ -30,6 +30,7 @@ log = { workspace = true } clap = { workspace = true } dashmap = { workspace = true } itertools = { workspace = true } +tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["std", "env-filter"] } native-tls = { workspace = true } prometheus = { workspace = true } diff --git a/quic-forward-proxy/src/proxy_request_format.rs b/quic-forward-proxy/src/proxy_request_format.rs index 15b7af3d..5d2acafc 100644 --- a/quic-forward-proxy/src/proxy_request_format.rs +++ b/quic-forward-proxy/src/proxy_request_format.rs @@ -37,7 +37,7 @@ impl Display for TpuForwardingRequest { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "TpuForwardingRequest t9 {} tpu nodes", + "TpuForwardingRequest to {} tpu nodes", &self.tpu_nodes.len(), ) } diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 37f294e0..41824448 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{SocketAddr, UdpSocket}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; @@ -11,6 +11,9 @@ use itertools::Itertools; use log::{debug, info, trace, warn}; use quinn::{ClientConfig, Endpoint, EndpointConfig, TokioRuntime, TransportConfig, VarInt}; use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::Keypair; +use solana_sdk::transaction::Transaction; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use tokio::sync::broadcast::error::TryRecvError; use tokio::sync::{broadcast::Receiver, RwLock}; @@ -19,6 +22,7 @@ use solana_lite_rpc_core::proxy_request_format::{TpuForwardingRequest, TxData}; use solana_lite_rpc_core::quic_connection_utils::{ QuicConnectionParameters, SkipServerVerification, }; +use solana_lite_rpc_core::solana_utils::SerializableTransaction; use crate::tpu_utils::quinn_auto_reconnect::AutoReconnect; @@ -249,4 +253,42 @@ impl QuicProxyConnectionManager { Ok(()) } + + + // testing only + pub async fn send_simple_transactions( + txs: Vec, + tpu_fanout_nodes: Vec, + proxy_address: SocketAddr, + ) -> anyhow::Result<()> { + + let identity = Keypair::new(); + + let (certificate, key) = new_self_signed_tls_certificate( + &identity, + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + ).unwrap(); + + let txs_data = txs.iter() + .map(|tx| { + let tx_raw = bincode::serialize(tx).unwrap(); + let sig = tx.get_signature().to_string(); + TxData::new(sig, tx_raw) + }) + .collect_vec(); + + + let endpoint = Self::create_proxy_client_endpoint(certificate, key); + let auto_reconnect = AutoReconnect::new(endpoint, proxy_address); + + info!("Sending {} transactions to quic proxy:", txs_data.len()); + for tx in &txs_data { + info!("- {:?}", tx) + } + + let result = Self::send_copy_of_txs_to_quicproxy( + txs_data.as_slice(), &auto_reconnect, proxy_address, tpu_fanout_nodes).await; + + result + } } From faa3effd21cad34637aa48fcb4d523f0d0d77c7f Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 14:56:23 +0200 Subject: [PATCH 20/42] send_test_proxy_forward_requests --- .../bin/send_test_proxy_forward_requests.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs diff --git a/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs b/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs new file mode 100644 index 00000000..65c24fa1 --- /dev/null +++ b/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs @@ -0,0 +1,49 @@ +use std::net::SocketAddr; +use std::str::FromStr; +use solana_sdk::pubkey::Pubkey; +use solana_sdk::signature::{Keypair, Signer}; +use solana_sdk::transaction::Transaction; +use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::{QuicProxyConnectionManager, TpuNode}; + +/// +/// test with test-validator and quic-proxy +/// +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +pub async fn main() { + tracing_subscriber::fmt::fmt() + .with_max_level(tracing::Level::DEBUG).init(); + + send_test_to_proxy().await; +} + +async fn send_test_to_proxy() { + + let tx = create_tx().await; + + // note: use the propaged TPU port - NOT port+6 + let tpu_address: SocketAddr = "127.0.0.1:1027".parse().unwrap(); + let proxy_address: SocketAddr = "127.0.0.1:11111".parse().unwrap(); + + let tpu_node = TpuNode { + tpu_address, + // tpu pubkey is only used for logging + tpu_identity: Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ").unwrap(), + }; + QuicProxyConnectionManager::send_simple_transactions(vec![tx], vec![tpu_node], proxy_address).await.unwrap(); + + +} + + +async fn create_tx() -> Transaction { + let payer = Keypair::from_base58_string( + "rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr", + ); + let payer_pubkey = payer.pubkey(); + + let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&payer_pubkey]); + + let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); + + tx +} From e86260623cdf399610013617e6a7bb7fae75002c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 15:02:26 +0200 Subject: [PATCH 21/42] continue on broadcast error --- services/src/tpu_utils/tpu_connection_manager.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/src/tpu_utils/tpu_connection_manager.rs b/services/src/tpu_utils/tpu_connection_manager.rs index 3c6b13be..b0eebd42 100644 --- a/services/src/tpu_utils/tpu_connection_manager.rs +++ b/services/src/tpu_utils/tpu_connection_manager.rs @@ -133,7 +133,7 @@ impl ActiveConnection { "Broadcast channel error on recv for {} error {} - continue", identity, e ); - break; + continue; } }; From ede7d6256962f8855407b926e01093303742e6cf Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 15:02:47 +0200 Subject: [PATCH 22/42] format --- quic-forward-proxy/Cargo.toml | 2 -- 1 file changed, 2 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 22b1b039..95dddc80 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -9,8 +9,6 @@ repository = "https://github.com/blockworks-foundation/lite-rpc" license = "AGPL" publish = false - - [dependencies] solana-lite-rpc-core = { workspace = true } solana-sdk = { workspace = true } From a3ed4d3561ff38fcdf90bb1fe40f4c91e1cdc0f5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 15:16:12 +0200 Subject: [PATCH 23/42] add bench for direct quic --- bench/Cargo.toml | 8 + bench/src/{main.rs => bench_direct_quic.rs} | 109 +++--- bench/src/bench_rpc.rs | 356 ++++++++++++++++++++ 3 files changed, 429 insertions(+), 44 deletions(-) rename bench/src/{main.rs => bench_direct_quic.rs} (83%) create mode 100644 bench/src/bench_rpc.rs diff --git a/bench/Cargo.toml b/bench/Cargo.toml index ceba95d9..7ecaf9eb 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -3,6 +3,14 @@ name = "bench" version = "0.2.3" edition = "2021" +[[bin]] +name = "bench-rpc" +path = "src/bench_rpc.rs" + +[[bin]] +name = "bench-direct-quic" +path = "src/bench_direct_quic.rs" + [dependencies] solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } solana-sdk = { workspace = true } diff --git a/bench/src/main.rs b/bench/src/bench_direct_quic.rs similarity index 83% rename from bench/src/main.rs rename to bench/src/bench_direct_quic.rs index 1a34db6d..a59a79e7 100644 --- a/bench/src/main.rs +++ b/bench/src/bench_direct_quic.rs @@ -1,7 +1,4 @@ -use std::fs; -use std::fs::{File, read_to_string}; -use std::net::{SocketAddr, SocketAddrV4}; -use std::str::FromStr; +use anyhow::Context; use bench::{ cli::Args, helpers::BenchHelper, @@ -12,32 +9,42 @@ use dashmap::DashMap; use futures::future::join_all; use log::{debug, error, info}; use solana_lite_rpc_core::tx_store::TxStore; -use solana_lite_rpc_core::{structures::identity_stakes::IdentityStakesData, quic_connection_utils::QuicConnectionParameters}; +use solana_lite_rpc_core::{ + quic_connection_utils::QuicConnectionParameters, + structures::identity_stakes::IdentityStakesData, +}; +use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; +use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; +use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_sdk::{ - commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, - slot_history::Slot, pubkey::Pubkey, -}; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; -use std::{sync::{ - atomic::{AtomicU64, Ordering}, - Arc, -}, net::{IpAddr, Ipv4Addr}, collections::HashMap}; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::signature::Signature; use solana_sdk::transaction::Transaction; +use solana_sdk::{ + commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, signature::Keypair, + signer::Signer, slot_history::Slot, +}; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::fs; +use std::fs::read_to_string; +use std::net::{SocketAddr, SocketAddrV4}; +use std::str::FromStr; use std::sync::atomic::AtomicBool; use std::time::SystemTime; -use anyhow::Context; +use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; use tracing_subscriber::fmt::format; -use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; -use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; -use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] async fn main() { @@ -76,15 +83,16 @@ async fn main() { let identity = Keypair::new(); - let (certificate, key) = new_self_signed_tls_certificate( - &identity, - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ) - .expect("Failed to initialize QUIC client certificates"); + let (certificate, key) = + new_self_signed_tls_certificate(&identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) + .expect("Failed to initialize QUIC client certificates"); let tpu_connection_manager = TpuConnectionManager::new(certificate, key, 4).await; let mut connections_to_keep = HashMap::new(); - connections_to_keep.insert(Pubkey::new_unique(), std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033)); + connections_to_keep.insert( + Pubkey::new_unique(), + std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033), + ); let identity_stakes = IdentityStakesData::default(); let tx_store = TxStore::default(); let connection_parameters = QuicConnectionParameters { @@ -98,7 +106,15 @@ async fn main() { }; let forwarder_channel = Arc::new(forwarder_channel); - tpu_connection_manager.update_connections(forwarder_channel.clone(), connections_to_keep, identity_stakes, tx_store, connection_parameters).await; + tpu_connection_manager + .update_connections( + forwarder_channel.clone(), + connections_to_keep, + identity_stakes, + tx_store, + connection_parameters, + ) + .await; { // block hash updater task @@ -202,7 +218,7 @@ async fn bench( current_slot: Arc, tx_metric_sx: UnboundedSender, log_txs: bool, - forwarder_channel : Arc)>> + forwarder_channel: Arc)>>, ) -> Metric { let map_of_txs: Arc> = Arc::new(DashMap::new()); let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); @@ -214,7 +230,6 @@ async fn bench( validator_identity, forward_receiver, exit_signal, - )); } // transaction sender task @@ -226,7 +241,6 @@ async fn bench( let map_of_txs = map_of_txs.clone(); let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); for rand_string in rand_strings { - let blockhash = { *block_hash.read().await }; let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); @@ -250,24 +264,27 @@ async fn bench( // } // let tpu_address = "127.0.0.1:1033".parse().unwrap(); - debug!("sent tx {} to {} tpu nodes", tx.get_signature(), leader_addrs.len()); + debug!( + "sent tx {} to {} tpu nodes", + tx.get_signature(), + leader_addrs.len() + ); for tpu_address in &leader_addrs { let tx_raw = bincode::serialize::(&tx).unwrap(); - let packet = ForwardPacket::new( - vec![tx_raw], - SocketAddr::from(*tpu_address), - 424242, - ); + let packet = + ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 424242); forwarder_channel.send(packet).await; - map_of_txs.insert(tx.get_signature().clone(), TxSendData { - sent_duration: start_time.elapsed(), - sent_instant: Instant::now(), - sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - }); + map_of_txs.insert( + tx.get_signature().clone(), + TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }, + ); } - } }); } @@ -319,17 +336,21 @@ async fn bench( metric } - fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); let file_age = SystemTime::now().duration_since(last_modified).unwrap(); - assert!(file_age.as_millis() < 1000, "leaders.dat is outdated ({:?}) - pls run patched lite-rpc service", file_age); + assert!( + file_age.as_millis() < 1000, + "leaders.dat is outdated ({:?}) - pls run patched lite-rpc service", + file_age + ); let leader_file = read_to_string(leaders_file)?; let mut leader_addrs = vec![]; for line in leader_file.lines() { - let socket_addr = SocketAddrV4::from_str(line).context(format!("error parsing line: {}", line)).unwrap(); + let socket_addr = SocketAddrV4::from_str(line) + .context(format!("error parsing line: {}", line)) + .unwrap(); leader_addrs.push(socket_addr); } Ok(leader_addrs) } - diff --git a/bench/src/bench_rpc.rs b/bench/src/bench_rpc.rs new file mode 100644 index 00000000..a59a79e7 --- /dev/null +++ b/bench/src/bench_rpc.rs @@ -0,0 +1,356 @@ +use anyhow::Context; +use bench::{ + cli::Args, + helpers::BenchHelper, + metrics::{AvgMetric, Metric, TxMetricData}, +}; +use clap::Parser; +use dashmap::DashMap; +use futures::future::join_all; +use log::{debug, error, info}; +use solana_lite_rpc_core::tx_store::TxStore; +use solana_lite_rpc_core::{ + quic_connection_utils::QuicConnectionParameters, + structures::identity_stakes::IdentityStakesData, +}; +use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; +use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; +use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; +use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_rpc_client::rpc_client::SerializableTransaction; +use solana_sdk::signature::Signature; +use solana_sdk::transaction::Transaction; +use solana_sdk::{ + commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, signature::Keypair, + signer::Signer, slot_history::Slot, +}; +use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::fs; +use std::fs::read_to_string; +use std::net::{SocketAddr, SocketAddrV4}; +use std::str::FromStr; +use std::sync::atomic::AtomicBool; +use std::time::SystemTime; +use std::{ + collections::HashMap, + net::{IpAddr, Ipv4Addr}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; +use tokio::{ + sync::{mpsc::UnboundedSender, RwLock}, + time::{Duration, Instant}, +}; +use tracing_subscriber::fmt::format; + +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +async fn main() { + tracing_subscriber::fmt::init(); + + let Args { + tx_count, + runs, + run_interval_ms, + metrics_file_name, + lite_rpc_addr, + transaction_save_file, + } = Args::parse(); + + let mut run_interval_ms = tokio::time::interval(Duration::from_millis(run_interval_ms)); + + info!("Connecting to {lite_rpc_addr}"); + + let mut avg_metric = AvgMetric::default(); + + let mut tasks = vec![]; + + let funded_payer = BenchHelper::get_payer().await.unwrap(); + println!("payer : {}", funded_payer.pubkey()); + + let rpc_client = Arc::new(RpcClient::new_with_commitment( + lite_rpc_addr.clone(), + CommitmentConfig::confirmed(), + )); + let bh = rpc_client.get_latest_blockhash().await.unwrap(); + let slot = rpc_client.get_slot().await.unwrap(); + let block_hash: Arc> = Arc::new(RwLock::new(bh)); + let current_slot = Arc::new(AtomicU64::new(slot)); + + let (forwarder_channel, _) = tokio::sync::broadcast::channel(1000); + + let identity = Keypair::new(); + + let (certificate, key) = + new_self_signed_tls_certificate(&identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) + .expect("Failed to initialize QUIC client certificates"); + + let tpu_connection_manager = TpuConnectionManager::new(certificate, key, 4).await; + let mut connections_to_keep = HashMap::new(); + connections_to_keep.insert( + Pubkey::new_unique(), + std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033), + ); + let identity_stakes = IdentityStakesData::default(); + let tx_store = TxStore::default(); + let connection_parameters = QuicConnectionParameters { + write_timeout: Duration::from_millis(500), + connection_retry_count: 10, + connection_timeout: Duration::from_millis(1000), + finalize_timeout: Duration::from_millis(100), + max_number_of_connections: 8, + number_of_transactions_per_unistream: 1, + unistream_timeout: Duration::from_millis(500), + }; + + let forwarder_channel = Arc::new(forwarder_channel); + tpu_connection_manager + .update_connections( + forwarder_channel.clone(), + connections_to_keep, + identity_stakes, + tx_store, + connection_parameters, + ) + .await; + + { + // block hash updater task + let block_hash = block_hash.clone(); + let rpc_client = rpc_client.clone(); + let current_slot = current_slot.clone(); + tokio::spawn(async move { + loop { + let bh = rpc_client.get_latest_blockhash().await; + match bh { + Ok(bh) => { + let mut lock = block_hash.write().await; + *lock = bh; + } + Err(e) => println!("blockhash update error {}", e), + } + + let slot = rpc_client.get_slot().await; + match slot { + Ok(slot) => { + current_slot.store(slot, std::sync::atomic::Ordering::Relaxed); + } + Err(e) => println!("slot {}", e), + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + }; + + // transaction logger + let (tx_log_sx, mut tx_log_rx) = tokio::sync::mpsc::unbounded_channel::(); + let log_transactions = !transaction_save_file.is_empty(); + if log_transactions { + tokio::spawn(async move { + let mut tx_writer = csv::Writer::from_path(transaction_save_file).unwrap(); + while let Some(x) = tx_log_rx.recv().await { + tx_writer.serialize(x).unwrap(); + } + }); + } + + for seed in 0..runs { + let funded_payer = Keypair::from_bytes(funded_payer.to_bytes().as_slice()).unwrap(); + tasks.push(tokio::spawn(bench( + rpc_client.clone(), + tx_count, + funded_payer, + seed as u64, + block_hash.clone(), + current_slot.clone(), + tx_log_sx.clone(), + log_transactions, + forwarder_channel.clone(), + ))); + // wait for an interval + run_interval_ms.tick().await; + } + + let join_res = join_all(tasks).await; + + let mut run_num = 1; + + let mut csv_writer = csv::Writer::from_path(metrics_file_name).unwrap(); + for res in join_res { + match res { + Ok(metric) => { + info!("Run {run_num}: Sent and Confirmed {tx_count} tx(s) in {metric:?} with",); + // update avg metric + avg_metric += &metric; + csv_writer.serialize(metric).unwrap(); + } + Err(_) => { + error!("join error for run {}", run_num); + } + } + run_num += 1; + } + + let avg_metric = Metric::from(avg_metric); + + info!("Avg Metric {avg_metric:?}",); + csv_writer.serialize(avg_metric).unwrap(); + + csv_writer.flush().unwrap(); +} + +#[derive(Clone, Debug, Copy)] +struct TxSendData { + sent_duration: Duration, + sent_instant: Instant, + sent_slot: Slot, +} + +#[allow(clippy::too_many_arguments)] +async fn bench( + rpc_client: Arc, + tx_count: usize, + funded_payer: Keypair, + seed: u64, + block_hash: Arc>, + current_slot: Arc, + tx_metric_sx: UnboundedSender, + log_txs: bool, + forwarder_channel: Arc)>>, +) -> Metric { + let map_of_txs: Arc> = Arc::new(DashMap::new()); + let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); + + { + let validator_identity = ValidatorIdentity::new(None); + let exit_signal = Arc::new(AtomicBool::new(false)); + let _jh = tokio::spawn(tx_forwarder( + validator_identity, + forward_receiver, + exit_signal, + )); + } + // transaction sender task + { + let map_of_txs = map_of_txs.clone(); + let current_slot = current_slot.clone(); + let forwarder_channel = forwarder_channel.clone(); + tokio::spawn(async move { + let map_of_txs = map_of_txs.clone(); + let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); + for rand_string in rand_strings { + let blockhash = { *block_hash.read().await }; + let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); + + let leader_addrs = read_leaders_from_file("leaders.dat").expect("leaders.dat file"); + + let start_time = Instant::now(); + // match rpc_client.send_transaction(&tx).await { + // Ok(signature) => { + // map_of_txs.insert( + // signature, + // TxSendData { + // sent_duration: start_time.elapsed(), + // sent_instant: Instant::now(), + // sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + // }, + // ); + // } + // Err(e) => { + // warn!("tx send failed with error {}", e); + // } + // } + // let tpu_address = "127.0.0.1:1033".parse().unwrap(); + + debug!( + "sent tx {} to {} tpu nodes", + tx.get_signature(), + leader_addrs.len() + ); + for tpu_address in &leader_addrs { + let tx_raw = bincode::serialize::(&tx).unwrap(); + let packet = + ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 424242); + + forwarder_channel.send(packet).await; + + map_of_txs.insert( + tx.get_signature().clone(), + TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }, + ); + } + } + }); + } + + let mut metric = Metric::default(); + let confirmation_time = Instant::now(); + let mut confirmed_count = 0; + while confirmation_time.elapsed() < Duration::from_secs(60) + && !(map_of_txs.is_empty() && confirmed_count == tx_count) + { + let signatures = map_of_txs.iter().map(|x| *x.key()).collect::>(); + if signatures.is_empty() { + tokio::time::sleep(Duration::from_millis(1)).await; + continue; + } + let chunks = signatures.chunks(100).collect::>(); + for chunk in chunks { + if let Ok(res) = rpc_client.get_signature_statuses(&chunk).await { + for (i, signature) in chunk.iter().enumerate() { + let tx_status = &res.value[i]; + if tx_status.is_some() { + let tx_data = map_of_txs.get(signature).unwrap(); + let time_to_confirm = tx_data.sent_instant.elapsed(); + metric.add_successful_transaction(tx_data.sent_duration, time_to_confirm); + + if log_txs { + let _ = tx_metric_sx.send(TxMetricData { + signature: signature.to_string(), + sent_slot: tx_data.sent_slot, + confirmed_slot: current_slot.load(Ordering::Relaxed), + time_to_send_in_millis: tx_data.sent_duration.as_millis() as u64, + time_to_confirm_in_millis: time_to_confirm.as_millis() as u64, + }); + } + drop(tx_data); + map_of_txs.remove(signature); + confirmed_count += 1; + } + } + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + } + + for tx in map_of_txs.iter() { + metric.add_unsuccessful_transaction(tx.sent_duration); + } + metric.finalize(); + metric +} + +fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { + let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); + let file_age = SystemTime::now().duration_since(last_modified).unwrap(); + assert!( + file_age.as_millis() < 1000, + "leaders.dat is outdated ({:?}) - pls run patched lite-rpc service", + file_age + ); + let leader_file = read_to_string(leaders_file)?; + let mut leader_addrs = vec![]; + for line in leader_file.lines() { + let socket_addr = SocketAddrV4::from_str(line) + .context(format!("error parsing line: {}", line)) + .unwrap(); + leader_addrs.push(socket_addr); + } + Ok(leader_addrs) +} From 4e8ded3a6dfd565d21419a96ca47148e0526d020 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 15:17:22 +0200 Subject: [PATCH 24/42] add 2nd bench bin --- bench/Cargo.toml | 4 - bench/src/main.rs | 233 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 233 insertions(+), 4 deletions(-) create mode 100644 bench/src/main.rs diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 7ecaf9eb..46cf680e 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -3,10 +3,6 @@ name = "bench" version = "0.2.3" edition = "2021" -[[bin]] -name = "bench-rpc" -path = "src/bench_rpc.rs" - [[bin]] name = "bench-direct-quic" path = "src/bench_direct_quic.rs" diff --git a/bench/src/main.rs b/bench/src/main.rs new file mode 100644 index 00000000..3b278bda --- /dev/null +++ b/bench/src/main.rs @@ -0,0 +1,233 @@ +use bench::{ + cli::Args, + helpers::BenchHelper, + metrics::{AvgMetric, Metric, TxMetricData}, +}; +use clap::Parser; +use dashmap::DashMap; +use futures::future::join_all; +use log::{error, info, warn}; +use solana_rpc_client::nonblocking::rpc_client::RpcClient; +use solana_sdk::{ + commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, + slot_history::Slot, +}; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; +use tokio::{ + sync::{mpsc::UnboundedSender, RwLock}, + time::{Duration, Instant}, +}; + +#[tokio::main(flavor = "multi_thread", worker_threads = 16)] +async fn main() { + tracing_subscriber::fmt::init(); + + let Args { + tx_count, + runs, + run_interval_ms, + metrics_file_name, + lite_rpc_addr, + transaction_save_file, + } = Args::parse(); + + let mut run_interval_ms = tokio::time::interval(Duration::from_millis(run_interval_ms)); + + info!("Connecting to {lite_rpc_addr}"); + + let mut avg_metric = AvgMetric::default(); + + let mut tasks = vec![]; + + let funded_payer = BenchHelper::get_payer().await.unwrap(); + println!("payer : {}", funded_payer.pubkey()); + + let rpc_client = Arc::new(RpcClient::new_with_commitment( + lite_rpc_addr.clone(), + CommitmentConfig::confirmed(), + )); + let bh = rpc_client.get_latest_blockhash().await.unwrap(); + let slot = rpc_client.get_slot().await.unwrap(); + let block_hash: Arc> = Arc::new(RwLock::new(bh)); + let current_slot = Arc::new(AtomicU64::new(slot)); + { + // block hash updater task + let block_hash = block_hash.clone(); + let rpc_client = rpc_client.clone(); + let current_slot = current_slot.clone(); + tokio::spawn(async move { + loop { + let bh = rpc_client.get_latest_blockhash().await; + match bh { + Ok(bh) => { + let mut lock = block_hash.write().await; + *lock = bh; + } + Err(e) => println!("blockhash update error {}", e), + } + + let slot = rpc_client.get_slot().await; + match slot { + Ok(slot) => { + current_slot.store(slot, std::sync::atomic::Ordering::Relaxed); + } + Err(e) => println!("slot {}", e), + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + }) + }; + + // transaction logger + let (tx_log_sx, mut tx_log_rx) = tokio::sync::mpsc::unbounded_channel::(); + let log_transactions = !transaction_save_file.is_empty(); + if log_transactions { + tokio::spawn(async move { + let mut tx_writer = csv::Writer::from_path(transaction_save_file).unwrap(); + while let Some(x) = tx_log_rx.recv().await { + tx_writer.serialize(x).unwrap(); + } + }); + } + + for seed in 0..runs { + let funded_payer = Keypair::from_bytes(funded_payer.to_bytes().as_slice()).unwrap(); + tasks.push(tokio::spawn(bench( + rpc_client.clone(), + tx_count, + funded_payer, + seed as u64, + block_hash.clone(), + current_slot.clone(), + tx_log_sx.clone(), + log_transactions, + ))); + // wait for an interval + run_interval_ms.tick().await; + } + + let join_res = join_all(tasks).await; + + let mut run_num = 1; + + let mut csv_writer = csv::Writer::from_path(metrics_file_name).unwrap(); + for res in join_res { + match res { + Ok(metric) => { + info!("Run {run_num}: Sent and Confirmed {tx_count} tx(s) in {metric:?} with",); + // update avg metric + avg_metric += &metric; + csv_writer.serialize(metric).unwrap(); + } + Err(_) => { + error!("join error for run {}", run_num); + } + } + run_num += 1; + } + + let avg_metric = Metric::from(avg_metric); + + info!("Avg Metric {avg_metric:?}",); + csv_writer.serialize(avg_metric).unwrap(); + + csv_writer.flush().unwrap(); +} + +#[derive(Clone, Debug, Copy)] +struct TxSendData { + sent_duration: Duration, + sent_instant: Instant, + sent_slot: Slot, +} + +#[allow(clippy::too_many_arguments)] +async fn bench( + rpc_client: Arc, + tx_count: usize, + funded_payer: Keypair, + seed: u64, + block_hash: Arc>, + current_slot: Arc, + tx_metric_sx: UnboundedSender, + log_txs: bool, +) -> Metric { + let map_of_txs = Arc::new(DashMap::new()); + // transaction sender task + { + let map_of_txs = map_of_txs.clone(); + let rpc_client = rpc_client.clone(); + let current_slot = current_slot.clone(); + tokio::spawn(async move { + let map_of_txs = map_of_txs.clone(); + let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); + + for rand_string in rand_strings { + let blockhash = { *block_hash.read().await }; + let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); + let start_time = Instant::now(); + match rpc_client.send_transaction(&tx).await { + Ok(signature) => { + map_of_txs.insert( + signature, + TxSendData { + sent_duration: start_time.elapsed(), + sent_instant: Instant::now(), + sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), + }, + ); + } + Err(e) => { + warn!("tx send failed with error {}", e); + } + } + } + }); + } + + let mut metric = Metric::default(); + let confirmation_time = Instant::now(); + let mut confirmed_count = 0; + while confirmation_time.elapsed() < Duration::from_secs(60) + && !(map_of_txs.is_empty() && confirmed_count == tx_count) + { + let signatures = map_of_txs.iter().map(|x| *x.key()).collect::>(); + if signatures.is_empty() { + tokio::time::sleep(Duration::from_millis(1)).await; + continue; + } + + if let Ok(res) = rpc_client.get_signature_statuses(&signatures).await { + for (i, signature) in signatures.iter().enumerate() { + let tx_status = &res.value[i]; + if tx_status.is_some() { + let tx_data = map_of_txs.get(signature).unwrap(); + let time_to_confirm = tx_data.sent_instant.elapsed(); + metric.add_successful_transaction(tx_data.sent_duration, time_to_confirm); + + if log_txs { + let _ = tx_metric_sx.send(TxMetricData { + signature: signature.to_string(), + sent_slot: tx_data.sent_slot, + confirmed_slot: current_slot.load(Ordering::Relaxed), + time_to_send_in_millis: tx_data.sent_duration.as_millis() as u64, + time_to_confirm_in_millis: time_to_confirm.as_millis() as u64, + }); + } + drop(tx_data); + map_of_txs.remove(signature); + confirmed_count += 1; + } + } + } + } + + for tx in map_of_txs.iter() { + metric.add_unsuccessful_transaction(tx.sent_duration); + } + metric.finalize(); + metric +} From 702b908a1039b5f8489542b897b12cce202443c2 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 17:39:20 +0200 Subject: [PATCH 25/42] proxy: integrate prometheus --- quic-forward-proxy/Cargo.toml | 2 ++ quic-forward-proxy/src/outbound/tx_forward.rs | 8 ++++++++ quic-forward-proxy/src/proxy.rs | 11 +++++++++- .../src/tpu_utils/quinn_auto_reconnect.rs | 20 ++++++------------- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/quic-forward-proxy/Cargo.toml b/quic-forward-proxy/Cargo.toml index 95dddc80..dc373154 100644 --- a/quic-forward-proxy/Cargo.toml +++ b/quic-forward-proxy/Cargo.toml @@ -11,6 +11,8 @@ publish = false [dependencies] solana-lite-rpc-core = { workspace = true } +# required for promentheus-sync +solana-lite-rpc-services = { workspace = true } solana-sdk = { workspace = true } solana-streamer = { workspace = true } solana-transaction-status = { workspace = true } diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index fb8034ca..5177209c 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -19,6 +19,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; +use prometheus::{IntCounter, opts, register_int_counter}; use tokio::sync::mpsc::Receiver; use tokio::sync::RwLock; @@ -26,6 +27,12 @@ const MAX_PARALLEL_STREAMS: usize = 6; pub const PARALLEL_TPU_CONNECTION_COUNT: usize = 4; const AGENT_SHUTDOWN_IDLE: Duration = Duration::from_millis(2500); // ms; should be 4x400ms+buffer +lazy_static::lazy_static! { + static ref OUTBOUND_SEND_TX: IntCounter = + register_int_counter!(opts!("literpcproxy_send_tx", "Proxy to TPU send transaction")).unwrap(); +} + + struct AgentHandle { pub tpu_address: SocketAddr, pub agent_exit_signal: Arc, @@ -168,6 +175,7 @@ pub async fn tx_forwarder( match result { Ok(()) => { + OUTBOUND_SEND_TX.inc(); debug!("send_txs_to_tpu_static sent {}", transactions_batch.len()); debug!( "Outbound connection stats: {}", diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index de974fe8..3df32c2e 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use std::net::{SocketAddr, SocketAddrV4}; use anyhow::bail; use std::sync::atomic::AtomicBool; @@ -10,6 +10,7 @@ use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::AnyhowJoinHandle; use crate::validator_identity::ValidatorIdentity; use log::info; +use solana_lite_rpc_services::prometheus_sync::PrometheusSync; pub struct QuicForwardProxy { // endpoint: Endpoint, @@ -56,6 +57,11 @@ impl QuicForwardProxy { exit_signal_clone, )); + // TODO make it a cli arg + let prometheus_addr: SocketAddrV4 = "127.0.0.1:9092".parse().unwrap(); + + let prometheus = PrometheusSync::sync(prometheus_addr); + tokio::select! { res = quic_proxy => { bail!("TPU Quic Proxy server exited unexpectedly {res:?}"); @@ -63,6 +69,9 @@ impl QuicForwardProxy { res = forwarder => { bail!("TPU Quic Tx forwarder exited unexpectedly {res:?}"); }, + res = prometheus => { + bail!("Prometheus sync service exited unexpectedly {res:?}"); + } } } } diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 32f05367..964a3751 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -11,11 +11,11 @@ use tracing::debug; /// copy of quic-proxy AutoReconnect - used that for reference const SEND_TIMEOUT: Duration = Duration::from_secs(5); -const MAX_RETRY_ATTEMPTS: u32 = 10; enum ConnectionState { NotConnected, Connection(Connection), + // not used for ocnnection to proxy PermanentError, FailedAttempt(u32), } @@ -145,19 +145,11 @@ impl AutoReconnect { *lock = ConnectionState::Connection(new_connection); } None => { - if *attempts < MAX_RETRY_ATTEMPTS { - warn!( - "Reconnect to {} failed (attempt {})", - self.target_address, attempts - ); - *lock = ConnectionState::FailedAttempt(attempts + 1); - } else { - warn!( - "Reconnect to {} failed permanently (attempt {})", - self.target_address, attempts - ); - *lock = ConnectionState::PermanentError; - } + warn!( + "Reconnect to {} failed (attempt {})", + self.target_address, attempts + ); + *lock = ConnectionState::FailedAttempt(attempts + 1); } }; } From 1b10e1412f99ed5f2705d20f5c9b2789fbb31b65 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 17:40:44 +0200 Subject: [PATCH 26/42] clippy --- .../bin/send_test_proxy_forward_requests.rs | 23 ++++++++-------- .../tests/quic_proxy_tpu_integrationtest.rs | 1 - .../quic_proxy_connection_manager.rs | 26 +++++++++---------- services/src/tpu_utils/tpu_service.rs | 17 +++++------- 4 files changed, 31 insertions(+), 36 deletions(-) diff --git a/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs b/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs index 65c24fa1..be45b3db 100644 --- a/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs +++ b/quic-forward-proxy-integration-test/src/bin/send_test_proxy_forward_requests.rs @@ -1,9 +1,11 @@ -use std::net::SocketAddr; -use std::str::FromStr; +use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::{ + QuicProxyConnectionManager, TpuNode, +}; use solana_sdk::pubkey::Pubkey; use solana_sdk::signature::{Keypair, Signer}; use solana_sdk::transaction::Transaction; -use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::{QuicProxyConnectionManager, TpuNode}; +use std::net::SocketAddr; +use std::str::FromStr; /// /// test with test-validator and quic-proxy @@ -11,13 +13,13 @@ use solana_lite_rpc_services::tpu_utils::quic_proxy_connection_manager::{QuicPro #[tokio::main(flavor = "multi_thread", worker_threads = 16)] pub async fn main() { tracing_subscriber::fmt::fmt() - .with_max_level(tracing::Level::DEBUG).init(); + .with_max_level(tracing::Level::DEBUG) + .init(); send_test_to_proxy().await; } async fn send_test_to_proxy() { - let tx = create_tx().await; // note: use the propaged TPU port - NOT port+6 @@ -29,12 +31,11 @@ async fn send_test_to_proxy() { // tpu pubkey is only used for logging tpu_identity: Pubkey::from_str("1111111jepwNWbYG87sgwnBbUJnQHrPiUJzMpqJXZ").unwrap(), }; - QuicProxyConnectionManager::send_simple_transactions(vec![tx], vec![tpu_node], proxy_address).await.unwrap(); - - + QuicProxyConnectionManager::send_simple_transactions(vec![tx], vec![tpu_node], proxy_address) + .await + .unwrap(); } - async fn create_tx() -> Transaction { let payer = Keypair::from_base58_string( "rKiJ7H5UUp3JR18kNyTF1XPuwPKHEM7gMLWHZPWP5djrW1vSjfwjhvJrevxF9MPmUmN9gJMLHZdLMgc9ao78eKr", @@ -43,7 +44,5 @@ async fn create_tx() -> Transaction { let memo_ix = spl_memo::build_memo("Hello world".as_bytes(), &[&payer_pubkey]); - let tx = Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)); - - tx + Transaction::new_with_payer(&[memo_ix], Some(&payer_pubkey)) } diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index c0739cdc..50d249cd 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -184,7 +184,6 @@ fn wireup_and_send_txs_via_channel(test_case_params: TestCaseParams) { // lite-rpc let runtime_literpc = Builder::new_multi_thread() - // see lite-rpc -> main.rs .worker_threads(16) // also works with 1 .enable_all() .build() diff --git a/services/src/tpu_utils/quic_proxy_connection_manager.rs b/services/src/tpu_utils/quic_proxy_connection_manager.rs index 81a2704a..3ed4e216 100644 --- a/services/src/tpu_utils/quic_proxy_connection_manager.rs +++ b/services/src/tpu_utils/quic_proxy_connection_manager.rs @@ -1,5 +1,5 @@ use std::collections::HashMap; -use std::net::{SocketAddr, UdpSocket}; +use std::net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket}; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering::Relaxed; use std::sync::Arc; @@ -254,22 +254,20 @@ impl QuicProxyConnectionManager { Ok(()) } - // testing only pub async fn send_simple_transactions( txs: Vec, tpu_fanout_nodes: Vec, proxy_address: SocketAddr, ) -> anyhow::Result<()> { - let identity = Keypair::new(); - let (certificate, key) = new_self_signed_tls_certificate( - &identity, - IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), - ).unwrap(); + let (certificate, key) = + new_self_signed_tls_certificate(&identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) + .unwrap(); - let txs_data = txs.iter() + let txs_data = txs + .iter() .map(|tx| { let tx_raw = bincode::serialize(tx).unwrap(); let sig = tx.get_signature().to_string(); @@ -277,7 +275,6 @@ impl QuicProxyConnectionManager { }) .collect_vec(); - let endpoint = Self::create_proxy_client_endpoint(certificate, key); let auto_reconnect = AutoReconnect::new(endpoint, proxy_address); @@ -286,9 +283,12 @@ impl QuicProxyConnectionManager { info!("- {:?}", tx) } - let result = Self::send_copy_of_txs_to_quicproxy( - txs_data.as_slice(), &auto_reconnect, proxy_address, tpu_fanout_nodes).await; - - result + Self::send_copy_of_txs_to_quicproxy( + txs_data.as_slice(), + &auto_reconnect, + proxy_address, + tpu_fanout_nodes, + ) + .await } } diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 9a573438..23cfc503 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -5,23 +5,23 @@ use super::tpu_connection_manager::TpuConnectionManager; use crate::tpu_utils::quic_proxy_connection_manager::QuicProxyConnectionManager; use crate::tpu_utils::tpu_connection_path::TpuConnectionPath; use crate::tpu_utils::tpu_service::ConnectionManager::{DirectTpu, QuicProxy}; +use itertools::Itertools; use solana_lite_rpc_core::data_cache::DataCache; use solana_lite_rpc_core::leaders_fetcher_trait::LeaderFetcherInterface; use solana_lite_rpc_core::quic_connection_utils::QuicConnectionParameters; use solana_lite_rpc_core::streams::SlotStream; use solana_lite_rpc_core::AnyhowJoinHandle; +use solana_sdk::pubkey::Pubkey; use solana_sdk::{quic::QUIC_PORT_OFFSET, signature::Keypair, slot_history::Slot}; use solana_streamer::tls_certificates::new_self_signed_tls_certificate; +use std::collections::HashMap; +use std::fs::File; +use std::io::Write; +use std::net::SocketAddr; use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, }; -use std::collections::HashMap; -use std::fs::{File, read_to_string}; -use std::io::Write; -use std::net::SocketAddr; -use itertools::Itertools; -use solana_sdk::pubkey::Pubkey; use tokio::time::Duration; lazy_static::lazy_static! { @@ -152,11 +152,9 @@ impl TpuService { }) .collect(); - dump_leaders_to_file(connections_to_keep.values().collect_vec()); // read_file(); - match &self.connection_manager { DirectTpu { tpu_connection_manager, @@ -211,10 +209,9 @@ fn dump_leaders_to_file(leaders: Vec<&SocketAddr>) { // 69.197.20.37:8009 let mut out_file = File::create("leaders.dat.tmp").unwrap(); for leader_addr in &leaders { - write!(out_file, "{}\n", leader_addr).unwrap(); + writeln!(out_file, "{}", leader_addr).unwrap(); } out_file.flush().unwrap(); std::fs::rename("leaders.dat.tmp", "leaders.dat").unwrap(); } - From 6154043199a0469e22ca9cc544b4cff3f60ed402 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 18:38:07 +0200 Subject: [PATCH 27/42] prometheus config --- quic-forward-proxy/README.md | 6 +++++ quic-forward-proxy/src/cli.rs | 3 +++ quic-forward-proxy/src/main.rs | 10 +++++--- quic-forward-proxy/src/outbound/tx_forward.rs | 24 ++++++++++++++++--- quic-forward-proxy/src/proxy.rs | 9 ------- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/quic-forward-proxy/README.md b/quic-forward-proxy/README.md index 76ac7d8c..cbc01738 100644 --- a/quic-forward-proxy/README.md +++ b/quic-forward-proxy/README.md @@ -51,3 +51,9 @@ QUIC/QUINN Endpoint and Connection specifics --------------------------- * keep-alive and idle timeout: both values must be aligned AND they must be configured on both endpoints (see [docs](https://docs.rs/quinn/latest/quinn/struct.TransportConfig.html#method.keep_alive_interval)) * tune or disable __max_concurrent_uni_streams__ respectively + + +Monitoring +--------------------------- +The Quic Proxy exposes prometheus metrics on address configured using _prometheus_addr_. + diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index 4c9d1d97..e57f1a65 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -8,4 +8,7 @@ pub struct Args { // e.g. 0.0.0.0:11111 or "localhost:11111" #[arg(short = 'l', long, env)] pub proxy_listen_addr: String, + /// enable metrics to prometheus at addr + #[arg(short = 'm', long, default_value_t = String::from("[::]:9092"))] + pub prometheus_addr: String, } diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 729fbd2c..652f4315 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -7,6 +7,7 @@ use dotenv::dotenv; use log::info; use solana_lite_rpc_core::keypair_loader::load_identity_keypair; use std::sync::Arc; +use solana_lite_rpc_services::prometheus_sync::PrometheusSync; use crate::validator_identity::ValidatorIdentity; @@ -31,6 +32,7 @@ pub async fn main() -> anyhow::Result<()> { let Args { identity_keypair, proxy_listen_addr, + prometheus_addr, } = Args::parse(); dotenv().ok(); @@ -42,15 +44,17 @@ pub async fn main() -> anyhow::Result<()> { .await? .start_services(); + let prometheus = PrometheusSync::sync(prometheus_addr); + let ctrl_c_signal = tokio::signal::ctrl_c(); tokio::select! { res = main_services => { bail!("Services quit unexpectedly {res:?}"); }, - // res = test_client => { - // bail!("Test Client quit unexpectedly {res:?}"); - // }, + res = prometheus => { + bail!("Prometheus sync service exited unexpectedly {res:?}"); + } _ = ctrl_c_signal => { info!("Received ctrl+c signal"); diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 5177209c..273080bd 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -19,7 +19,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use prometheus::{IntCounter, opts, register_int_counter}; +use prometheus::{IntCounter, IntGauge, opts, register_int_counter, register_int_gauge}; use tokio::sync::mpsc::Receiver; use tokio::sync::RwLock; @@ -29,10 +29,15 @@ const AGENT_SHUTDOWN_IDLE: Duration = Duration::from_millis(2500); // ms; should lazy_static::lazy_static! { static ref OUTBOUND_SEND_TX: IntCounter = - register_int_counter!(opts!("literpcproxy_send_tx", "Proxy to TPU send transaction")).unwrap(); + register_int_counter!(opts!("literpcproxy_send_tx", "Proxy to TPU send transaction")).unwrap(); + static ref OUTBOUND_SEND_ERRORS: IntCounter = + register_int_counter!(opts!("literpcproxy_send_errors", "Proxy to TPU send errors")).unwrap(); + static ref OUTBOUND_BATCH_SIZE: IntGauge = + register_int_gauge!(opts!("literpcproxy_batch_size", "Proxy to TPU tx batch size")).unwrap(); + static ref OUTBOUND_PACKET_SIZE: IntGauge = + register_int_gauge!(opts!("literpcproxy_packet_size", "Proxy to TPU packet size")).unwrap(); } - struct AgentHandle { pub tpu_address: SocketAddr, pub agent_exit_signal: Arc, @@ -173,6 +178,10 @@ pub async fn tx_forwarder( auto_connection.target_address )); + OUTBOUND_BATCH_SIZE.set(transactions_batch.len() as i64); + + OUTBOUND_PACKET_SIZE.set(count_bytes(&transactions_batch)); + match result { Ok(()) => { OUTBOUND_SEND_TX.inc(); @@ -183,6 +192,7 @@ pub async fn tx_forwarder( ); } Err(err) => { + OUTBOUND_SEND_ERRORS.inc(); warn!("got send_txs_to_tpu_static error {} - loop over errors", err); } } @@ -224,6 +234,14 @@ pub async fn tx_forwarder( // not reachable } +fn count_bytes(tx_vec: &Vec>) -> i64 { + let mut total_bytes = 0; + for tx in tx_vec { + total_bytes += tx.len(); + } + total_bytes as i64 +} + async fn cleanup_agents( agents: &mut HashMap, current_tpu_address: &SocketAddr, diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 3df32c2e..79052ae1 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -10,7 +10,6 @@ use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::AnyhowJoinHandle; use crate::validator_identity::ValidatorIdentity; use log::info; -use solana_lite_rpc_services::prometheus_sync::PrometheusSync; pub struct QuicForwardProxy { // endpoint: Endpoint, @@ -57,11 +56,6 @@ impl QuicForwardProxy { exit_signal_clone, )); - // TODO make it a cli arg - let prometheus_addr: SocketAddrV4 = "127.0.0.1:9092".parse().unwrap(); - - let prometheus = PrometheusSync::sync(prometheus_addr); - tokio::select! { res = quic_proxy => { bail!("TPU Quic Proxy server exited unexpectedly {res:?}"); @@ -69,9 +63,6 @@ impl QuicForwardProxy { res = forwarder => { bail!("TPU Quic Tx forwarder exited unexpectedly {res:?}"); }, - res = prometheus => { - bail!("Prometheus sync service exited unexpectedly {res:?}"); - } } } } From 3e98c52ae4ee6f8e5e8e2c9a3b0dd854a59a98bb Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 19:28:37 +0200 Subject: [PATCH 28/42] add feature for "leaders.dat" --- Cargo.lock | 1 + bench/src/bench_direct_quic.rs | 88 +------ bench/src/bench_rpc.rs | 356 -------------------------- bench/src/cli_direct_quic.rs | 23 ++ lite-rpc/src/main.rs | 2 +- quic-forward-proxy/src/cli.rs | 2 +- services/src/tpu_utils/tpu_service.rs | 10 +- 7 files changed, 47 insertions(+), 435 deletions(-) delete mode 100644 bench/src/bench_rpc.rs create mode 100644 bench/src/cli_direct_quic.rs diff --git a/Cargo.lock b/Cargo.lock index 0fc90c11..73c3b4c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4337,6 +4337,7 @@ dependencies = [ "serde", "serde_json", "solana-lite-rpc-core", + "solana-lite-rpc-services", "solana-net-utils", "solana-sdk", "solana-streamer", diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index a59a79e7..d978ff59 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -1,6 +1,7 @@ +mod cli_direct_quic; + use anyhow::Context; use bench::{ - cli::Args, helpers::BenchHelper, metrics::{AvgMetric, Metric, TxMetricData}, }; @@ -8,24 +9,18 @@ use clap::Parser; use dashmap::DashMap; use futures::future::join_all; use log::{debug, error, info}; -use solana_lite_rpc_core::tx_store::TxStore; -use solana_lite_rpc_core::{ - quic_connection_utils::QuicConnectionParameters, - structures::identity_stakes::IdentityStakesData, -}; use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; -use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; + use solana_rpc_client::nonblocking::rpc_client::RpcClient; use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::signature::Signature; use solana_sdk::transaction::Transaction; use solana_sdk::{ - commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, signature::Keypair, + commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, slot_history::Slot, }; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; use std::fs; use std::fs::read_to_string; use std::net::{SocketAddr, SocketAddrV4}; @@ -33,8 +28,6 @@ use std::str::FromStr; use std::sync::atomic::AtomicBool; use std::time::SystemTime; use std::{ - collections::HashMap, - net::{IpAddr, Ipv4Addr}, sync::{ atomic::{AtomicU64, Ordering}, Arc, @@ -44,24 +37,23 @@ use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, }; -use tracing_subscriber::fmt::format; #[tokio::main(flavor = "multi_thread", worker_threads = 16)] async fn main() { tracing_subscriber::fmt::init(); - let Args { + let cli_direct_quic::Args { tx_count, runs, run_interval_ms, metrics_file_name, - lite_rpc_addr, + rpc_addr, transaction_save_file, - } = Args::parse(); + } = cli_direct_quic::Args::parse(); let mut run_interval_ms = tokio::time::interval(Duration::from_millis(run_interval_ms)); - info!("Connecting to {lite_rpc_addr}"); + info!("Using RPC service on {rpc_addr}"); let mut avg_metric = AvgMetric::default(); @@ -71,7 +63,7 @@ async fn main() { println!("payer : {}", funded_payer.pubkey()); let rpc_client = Arc::new(RpcClient::new_with_commitment( - lite_rpc_addr.clone(), + rpc_addr.clone(), CommitmentConfig::confirmed(), )); let bh = rpc_client.get_latest_blockhash().await.unwrap(); @@ -79,43 +71,6 @@ async fn main() { let block_hash: Arc> = Arc::new(RwLock::new(bh)); let current_slot = Arc::new(AtomicU64::new(slot)); - let (forwarder_channel, _) = tokio::sync::broadcast::channel(1000); - - let identity = Keypair::new(); - - let (certificate, key) = - new_self_signed_tls_certificate(&identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) - .expect("Failed to initialize QUIC client certificates"); - - let tpu_connection_manager = TpuConnectionManager::new(certificate, key, 4).await; - let mut connections_to_keep = HashMap::new(); - connections_to_keep.insert( - Pubkey::new_unique(), - std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033), - ); - let identity_stakes = IdentityStakesData::default(); - let tx_store = TxStore::default(); - let connection_parameters = QuicConnectionParameters { - write_timeout: Duration::from_millis(500), - connection_retry_count: 10, - connection_timeout: Duration::from_millis(1000), - finalize_timeout: Duration::from_millis(100), - max_number_of_connections: 8, - number_of_transactions_per_unistream: 1, - unistream_timeout: Duration::from_millis(500), - }; - - let forwarder_channel = Arc::new(forwarder_channel); - tpu_connection_manager - .update_connections( - forwarder_channel.clone(), - connections_to_keep, - identity_stakes, - tx_store, - connection_parameters, - ) - .await; - { // block hash updater task let block_hash = block_hash.clone(); @@ -167,7 +122,6 @@ async fn main() { current_slot.clone(), tx_log_sx.clone(), log_transactions, - forwarder_channel.clone(), ))); // wait for an interval run_interval_ms.tick().await; @@ -218,7 +172,6 @@ async fn bench( current_slot: Arc, tx_metric_sx: UnboundedSender, log_txs: bool, - forwarder_channel: Arc)>>, ) -> Metric { let map_of_txs: Arc> = Arc::new(DashMap::new()); let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); @@ -236,7 +189,7 @@ async fn bench( { let map_of_txs = map_of_txs.clone(); let current_slot = current_slot.clone(); - let forwarder_channel = forwarder_channel.clone(); + // let forwarder_channel = forwarder_channel.clone(); tokio::spawn(async move { let map_of_txs = map_of_txs.clone(); let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); @@ -247,22 +200,6 @@ async fn bench( let leader_addrs = read_leaders_from_file("leaders.dat").expect("leaders.dat file"); let start_time = Instant::now(); - // match rpc_client.send_transaction(&tx).await { - // Ok(signature) => { - // map_of_txs.insert( - // signature, - // TxSendData { - // sent_duration: start_time.elapsed(), - // sent_instant: Instant::now(), - // sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - // }, - // ); - // } - // Err(e) => { - // warn!("tx send failed with error {}", e); - // } - // } - // let tpu_address = "127.0.0.1:1033".parse().unwrap(); debug!( "sent tx {} to {} tpu nodes", @@ -272,9 +209,9 @@ async fn bench( for tpu_address in &leader_addrs { let tx_raw = bincode::serialize::(&tx).unwrap(); let packet = - ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 424242); + ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 0xdeadbeef); - forwarder_channel.send(packet).await; + forwarder_channel.send(packet).await.unwrap(); map_of_txs.insert( tx.get_signature().clone(), @@ -336,6 +273,7 @@ async fn bench( metric } +// note: this file gets written by tpu_services::dump_leaders_to_file; see docs how to activate fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); let file_age = SystemTime::now().duration_since(last_modified).unwrap(); diff --git a/bench/src/bench_rpc.rs b/bench/src/bench_rpc.rs deleted file mode 100644 index a59a79e7..00000000 --- a/bench/src/bench_rpc.rs +++ /dev/null @@ -1,356 +0,0 @@ -use anyhow::Context; -use bench::{ - cli::Args, - helpers::BenchHelper, - metrics::{AvgMetric, Metric, TxMetricData}, -}; -use clap::Parser; -use dashmap::DashMap; -use futures::future::join_all; -use log::{debug, error, info}; -use solana_lite_rpc_core::tx_store::TxStore; -use solana_lite_rpc_core::{ - quic_connection_utils::QuicConnectionParameters, - structures::identity_stakes::IdentityStakesData, -}; -use solana_lite_rpc_quic_forward_proxy::outbound::tx_forward::tx_forwarder; -use solana_lite_rpc_quic_forward_proxy::shared::ForwardPacket; -use solana_lite_rpc_quic_forward_proxy::validator_identity::ValidatorIdentity; -use solana_lite_rpc_services::tpu_utils::tpu_connection_manager::TpuConnectionManager; -use solana_rpc_client::nonblocking::rpc_client::RpcClient; -use solana_rpc_client::rpc_client::SerializableTransaction; -use solana_sdk::signature::Signature; -use solana_sdk::transaction::Transaction; -use solana_sdk::{ - commitment_config::CommitmentConfig, hash::Hash, pubkey::Pubkey, signature::Keypair, - signer::Signer, slot_history::Slot, -}; -use solana_streamer::tls_certificates::new_self_signed_tls_certificate; -use std::fs; -use std::fs::read_to_string; -use std::net::{SocketAddr, SocketAddrV4}; -use std::str::FromStr; -use std::sync::atomic::AtomicBool; -use std::time::SystemTime; -use std::{ - collections::HashMap, - net::{IpAddr, Ipv4Addr}, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, -}; -use tokio::{ - sync::{mpsc::UnboundedSender, RwLock}, - time::{Duration, Instant}, -}; -use tracing_subscriber::fmt::format; - -#[tokio::main(flavor = "multi_thread", worker_threads = 16)] -async fn main() { - tracing_subscriber::fmt::init(); - - let Args { - tx_count, - runs, - run_interval_ms, - metrics_file_name, - lite_rpc_addr, - transaction_save_file, - } = Args::parse(); - - let mut run_interval_ms = tokio::time::interval(Duration::from_millis(run_interval_ms)); - - info!("Connecting to {lite_rpc_addr}"); - - let mut avg_metric = AvgMetric::default(); - - let mut tasks = vec![]; - - let funded_payer = BenchHelper::get_payer().await.unwrap(); - println!("payer : {}", funded_payer.pubkey()); - - let rpc_client = Arc::new(RpcClient::new_with_commitment( - lite_rpc_addr.clone(), - CommitmentConfig::confirmed(), - )); - let bh = rpc_client.get_latest_blockhash().await.unwrap(); - let slot = rpc_client.get_slot().await.unwrap(); - let block_hash: Arc> = Arc::new(RwLock::new(bh)); - let current_slot = Arc::new(AtomicU64::new(slot)); - - let (forwarder_channel, _) = tokio::sync::broadcast::channel(1000); - - let identity = Keypair::new(); - - let (certificate, key) = - new_self_signed_tls_certificate(&identity, IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0))) - .expect("Failed to initialize QUIC client certificates"); - - let tpu_connection_manager = TpuConnectionManager::new(certificate, key, 4).await; - let mut connections_to_keep = HashMap::new(); - connections_to_keep.insert( - Pubkey::new_unique(), - std::net::SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 1033), - ); - let identity_stakes = IdentityStakesData::default(); - let tx_store = TxStore::default(); - let connection_parameters = QuicConnectionParameters { - write_timeout: Duration::from_millis(500), - connection_retry_count: 10, - connection_timeout: Duration::from_millis(1000), - finalize_timeout: Duration::from_millis(100), - max_number_of_connections: 8, - number_of_transactions_per_unistream: 1, - unistream_timeout: Duration::from_millis(500), - }; - - let forwarder_channel = Arc::new(forwarder_channel); - tpu_connection_manager - .update_connections( - forwarder_channel.clone(), - connections_to_keep, - identity_stakes, - tx_store, - connection_parameters, - ) - .await; - - { - // block hash updater task - let block_hash = block_hash.clone(); - let rpc_client = rpc_client.clone(); - let current_slot = current_slot.clone(); - tokio::spawn(async move { - loop { - let bh = rpc_client.get_latest_blockhash().await; - match bh { - Ok(bh) => { - let mut lock = block_hash.write().await; - *lock = bh; - } - Err(e) => println!("blockhash update error {}", e), - } - - let slot = rpc_client.get_slot().await; - match slot { - Ok(slot) => { - current_slot.store(slot, std::sync::atomic::Ordering::Relaxed); - } - Err(e) => println!("slot {}", e), - } - tokio::time::sleep(Duration::from_millis(100)).await; - } - }) - }; - - // transaction logger - let (tx_log_sx, mut tx_log_rx) = tokio::sync::mpsc::unbounded_channel::(); - let log_transactions = !transaction_save_file.is_empty(); - if log_transactions { - tokio::spawn(async move { - let mut tx_writer = csv::Writer::from_path(transaction_save_file).unwrap(); - while let Some(x) = tx_log_rx.recv().await { - tx_writer.serialize(x).unwrap(); - } - }); - } - - for seed in 0..runs { - let funded_payer = Keypair::from_bytes(funded_payer.to_bytes().as_slice()).unwrap(); - tasks.push(tokio::spawn(bench( - rpc_client.clone(), - tx_count, - funded_payer, - seed as u64, - block_hash.clone(), - current_slot.clone(), - tx_log_sx.clone(), - log_transactions, - forwarder_channel.clone(), - ))); - // wait for an interval - run_interval_ms.tick().await; - } - - let join_res = join_all(tasks).await; - - let mut run_num = 1; - - let mut csv_writer = csv::Writer::from_path(metrics_file_name).unwrap(); - for res in join_res { - match res { - Ok(metric) => { - info!("Run {run_num}: Sent and Confirmed {tx_count} tx(s) in {metric:?} with",); - // update avg metric - avg_metric += &metric; - csv_writer.serialize(metric).unwrap(); - } - Err(_) => { - error!("join error for run {}", run_num); - } - } - run_num += 1; - } - - let avg_metric = Metric::from(avg_metric); - - info!("Avg Metric {avg_metric:?}",); - csv_writer.serialize(avg_metric).unwrap(); - - csv_writer.flush().unwrap(); -} - -#[derive(Clone, Debug, Copy)] -struct TxSendData { - sent_duration: Duration, - sent_instant: Instant, - sent_slot: Slot, -} - -#[allow(clippy::too_many_arguments)] -async fn bench( - rpc_client: Arc, - tx_count: usize, - funded_payer: Keypair, - seed: u64, - block_hash: Arc>, - current_slot: Arc, - tx_metric_sx: UnboundedSender, - log_txs: bool, - forwarder_channel: Arc)>>, -) -> Metric { - let map_of_txs: Arc> = Arc::new(DashMap::new()); - let (forwarder_channel, forward_receiver) = tokio::sync::mpsc::channel(1000); - - { - let validator_identity = ValidatorIdentity::new(None); - let exit_signal = Arc::new(AtomicBool::new(false)); - let _jh = tokio::spawn(tx_forwarder( - validator_identity, - forward_receiver, - exit_signal, - )); - } - // transaction sender task - { - let map_of_txs = map_of_txs.clone(); - let current_slot = current_slot.clone(); - let forwarder_channel = forwarder_channel.clone(); - tokio::spawn(async move { - let map_of_txs = map_of_txs.clone(); - let rand_strings = BenchHelper::generate_random_strings(tx_count, Some(seed)); - for rand_string in rand_strings { - let blockhash = { *block_hash.read().await }; - let tx = BenchHelper::create_memo_tx(&rand_string, &funded_payer, blockhash); - - let leader_addrs = read_leaders_from_file("leaders.dat").expect("leaders.dat file"); - - let start_time = Instant::now(); - // match rpc_client.send_transaction(&tx).await { - // Ok(signature) => { - // map_of_txs.insert( - // signature, - // TxSendData { - // sent_duration: start_time.elapsed(), - // sent_instant: Instant::now(), - // sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - // }, - // ); - // } - // Err(e) => { - // warn!("tx send failed with error {}", e); - // } - // } - // let tpu_address = "127.0.0.1:1033".parse().unwrap(); - - debug!( - "sent tx {} to {} tpu nodes", - tx.get_signature(), - leader_addrs.len() - ); - for tpu_address in &leader_addrs { - let tx_raw = bincode::serialize::(&tx).unwrap(); - let packet = - ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 424242); - - forwarder_channel.send(packet).await; - - map_of_txs.insert( - tx.get_signature().clone(), - TxSendData { - sent_duration: start_time.elapsed(), - sent_instant: Instant::now(), - sent_slot: current_slot.load(std::sync::atomic::Ordering::Relaxed), - }, - ); - } - } - }); - } - - let mut metric = Metric::default(); - let confirmation_time = Instant::now(); - let mut confirmed_count = 0; - while confirmation_time.elapsed() < Duration::from_secs(60) - && !(map_of_txs.is_empty() && confirmed_count == tx_count) - { - let signatures = map_of_txs.iter().map(|x| *x.key()).collect::>(); - if signatures.is_empty() { - tokio::time::sleep(Duration::from_millis(1)).await; - continue; - } - let chunks = signatures.chunks(100).collect::>(); - for chunk in chunks { - if let Ok(res) = rpc_client.get_signature_statuses(&chunk).await { - for (i, signature) in chunk.iter().enumerate() { - let tx_status = &res.value[i]; - if tx_status.is_some() { - let tx_data = map_of_txs.get(signature).unwrap(); - let time_to_confirm = tx_data.sent_instant.elapsed(); - metric.add_successful_transaction(tx_data.sent_duration, time_to_confirm); - - if log_txs { - let _ = tx_metric_sx.send(TxMetricData { - signature: signature.to_string(), - sent_slot: tx_data.sent_slot, - confirmed_slot: current_slot.load(Ordering::Relaxed), - time_to_send_in_millis: tx_data.sent_duration.as_millis() as u64, - time_to_confirm_in_millis: time_to_confirm.as_millis() as u64, - }); - } - drop(tx_data); - map_of_txs.remove(signature); - confirmed_count += 1; - } - } - } - tokio::time::sleep(Duration::from_millis(50)).await; - } - } - - for tx in map_of_txs.iter() { - metric.add_unsuccessful_transaction(tx.sent_duration); - } - metric.finalize(); - metric -} - -fn read_leaders_from_file(leaders_file: &str) -> anyhow::Result> { - let last_modified = fs::metadata("leaders.dat")?.modified().unwrap(); - let file_age = SystemTime::now().duration_since(last_modified).unwrap(); - assert!( - file_age.as_millis() < 1000, - "leaders.dat is outdated ({:?}) - pls run patched lite-rpc service", - file_age - ); - let leader_file = read_to_string(leaders_file)?; - let mut leader_addrs = vec![]; - for line in leader_file.lines() { - let socket_addr = SocketAddrV4::from_str(line) - .context(format!("error parsing line: {}", line)) - .unwrap(); - leader_addrs.push(socket_addr); - } - Ok(leader_addrs) -} diff --git a/bench/src/cli_direct_quic.rs b/bench/src/cli_direct_quic.rs new file mode 100644 index 00000000..2038c534 --- /dev/null +++ b/bench/src/cli_direct_quic.rs @@ -0,0 +1,23 @@ +use clap::{command, Parser}; + +#[derive(Parser, Debug)] +#[command(author, version, about, long_about = None)] +pub struct Args { + /// Number of tx(s) sent in each run + #[arg(short = 'n', long, default_value_t = 5_000)] + pub tx_count: usize, + /// Number of bench runs + #[arg(short = 'r', long, default_value_t = 1)] + pub runs: usize, + /// Interval between each bench run (ms) + #[arg(short = 'i', long, default_value_t = 1000)] + pub run_interval_ms: u64, + /// Metrics output file name + #[arg(short = 'm', long, default_value_t = String::from("metrics.csv"))] + pub metrics_file_name: String, + /// Rpc Address to connect to + #[arg(long, default_value_t = String::from("http://127.0.0.1:8899"))] + pub rpc_addr: String, + #[arg(short = 't', long, default_value_t = String::from("transactions.csv"))] + pub transaction_save_file: String, +} diff --git a/lite-rpc/src/main.rs b/lite-rpc/src/main.rs index 7d32aa04..66ae0654 100644 --- a/lite-rpc/src/main.rs +++ b/lite-rpc/src/main.rs @@ -266,7 +266,7 @@ fn configure_tpu_connection_path(quic_proxy_addr: Option) -> TpuConnecti Some(prox_address) => { let proxy_socket_addr = parse_host_port(prox_address.as_str()).unwrap(); TpuConnectionPath::QuicForwardProxyPath { - // e.g. "127.0.0.1:11111" or "localhost:11111" + // e.g. "127.0.0.1:11111" forward_proxy_address: proxy_socket_addr, } } diff --git a/quic-forward-proxy/src/cli.rs b/quic-forward-proxy/src/cli.rs index e57f1a65..498a865a 100644 --- a/quic-forward-proxy/src/cli.rs +++ b/quic-forward-proxy/src/cli.rs @@ -5,7 +5,7 @@ use clap::Parser; pub struct Args { #[arg(short = 'k', long, default_value_t = String::new())] pub identity_keypair: String, - // e.g. 0.0.0.0:11111 or "localhost:11111" + // e.g. 0.0.0.0:11111 #[arg(short = 'l', long, env)] pub proxy_listen_addr: String, /// enable metrics to prometheus at addr diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 23cfc503..09155167 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -22,6 +22,7 @@ use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, }; +use std::path::Path; use tokio::time::Duration; lazy_static::lazy_static! { @@ -153,7 +154,6 @@ impl TpuService { .collect(); dump_leaders_to_file(connections_to_keep.values().collect_vec()); - // read_file(); match &self.connection_manager { DirectTpu { @@ -204,11 +204,17 @@ impl TpuService { } } +// used for bench tool +// optionally dumps the leaders to disk for later use fn dump_leaders_to_file(leaders: Vec<&SocketAddr>) { + // files acts as feature flag - create it on filesystem to enable + if !Path::exists(Path::new("leaders.dat")) { + return; + } // will create/truncate file - // 69.197.20.37:8009 let mut out_file = File::create("leaders.dat.tmp").unwrap(); for leader_addr in &leaders { + // 69.197.20.37:8009 writeln!(out_file, "{}", leader_addr).unwrap(); } out_file.flush().unwrap(); From 2bd60466133c5a0ec961305792d2407abe9085ef Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 19:29:00 +0200 Subject: [PATCH 29/42] cleanup --- quic-forward-proxy/src/proxy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quic-forward-proxy/src/proxy.rs b/quic-forward-proxy/src/proxy.rs index 79052ae1..de974fe8 100644 --- a/quic-forward-proxy/src/proxy.rs +++ b/quic-forward-proxy/src/proxy.rs @@ -1,4 +1,4 @@ -use std::net::{SocketAddr, SocketAddrV4}; +use std::net::SocketAddr; use anyhow::bail; use std::sync::atomic::AtomicBool; From f53511e53596ef6aab3880f479de57fc43f89115 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 19:41:29 +0200 Subject: [PATCH 30/42] add naive timeouts to .send_uni --- quic-forward-proxy/src/quinn_auto_reconnect.rs | 7 ++++--- services/src/tpu_utils/quinn_auto_reconnect.rs | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/quic-forward-proxy/src/quinn_auto_reconnect.rs b/quic-forward-proxy/src/quinn_auto_reconnect.rs index 14c0120c..e6147220 100644 --- a/quic-forward-proxy/src/quinn_auto_reconnect.rs +++ b/quic-forward-proxy/src/quinn_auto_reconnect.rs @@ -50,11 +50,12 @@ impl AutoReconnect { } pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { - let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) + let connection = self.refresh_and_get().await?; + let mut send_stream = timeout(SEND_TIMEOUT, connection.open_uni()) .await .context("open uni stream for sending")??; - send_stream.write_all(payload.as_slice()).await?; - send_stream.finish().await?; + timeout(SEND_TIMEOUT, send_stream.write_all(payload.as_slice())).await??; + timeout(SEND_TIMEOUT, send_stream.finish()).await??; Ok(()) } diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 964a3751..7e102e82 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -42,11 +42,12 @@ impl AutoReconnect { } pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { - let mut send_stream = timeout(SEND_TIMEOUT, self.refresh_and_get().await?.open_uni()) + let connection = self.refresh_and_get().await?; + let mut send_stream = timeout(SEND_TIMEOUT, connection.open_uni()) .await .context("open uni stream for sending")??; - send_stream.write_all(payload.as_slice()).await?; - send_stream.finish().await?; + timeout(SEND_TIMEOUT, send_stream.write_all(payload.as_slice())).await??; + timeout(SEND_TIMEOUT, send_stream.finish()).await??; Ok(()) } From 57c11cd4fddf2f15e8a1bb71b68a55aa328d35e1 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Mon, 18 Sep 2023 20:44:31 +0200 Subject: [PATCH 31/42] relax error handling --- .../src/inbound/proxy_listener.rs | 37 +++++++++++-------- quic-forward-proxy/src/outbound/tx_forward.rs | 9 +++-- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 49d5b17d..0a370e3a 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -133,18 +133,25 @@ impl ProxyListener { for tpu_node in proxy_request.get_tpu_nodes() { let tpu_address = tpu_node.tpu_socket_addr; - forwarder_channel_copy - .send_timeout( - ForwardPacket::new( - txs.clone(), - tpu_address, - proxy_request.get_hash(), - ), - FALLBACK_TIMEOUT, - ) - .await - .context("sending internal packet from proxy to forwarder") - .unwrap(); + + let send_result = + forwarder_channel_copy + .send_timeout( + ForwardPacket::new( + txs.clone(), + tpu_address, + proxy_request.get_hash(), + ), + FALLBACK_TIMEOUT, + ) + .await + .context("sending internal packet from proxy to forwarder"); + + if let Err(err) = send_result { + error!("send failed: {}", + err); + return; + } } }); @@ -156,10 +163,8 @@ impl ProxyListener { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { - return Err(anyhow!( - "connection closed by client with unexpected reason: {:?}", - reason - )); + bail!("connection closed by client with unexpected reason: {:?}", + reason); } debug!("connection gracefully closed by client"); return Ok(()); diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index 273080bd..dd10c7f2 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -226,9 +226,12 @@ pub async fn tx_forwarder( debug!("tx-forward queue len: {}", broadcast_in.len()) } - broadcast_in - .send(forward_packet) - .expect("send must succeed"); + let enqueue_result = broadcast_in + .send(forward_packet); + + if let Err(e) = enqueue_result { + warn!("broadcast channel send error: {}", e); + } } // -- loop over transactions from upstream channels // not reachable From 897a798987d8a5492c214654b594e1e7209282de Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 10:54:39 +0200 Subject: [PATCH 32/42] comment assumptions for connection manager --- .../src/tpu_utils/quinn_auto_reconnect.rs | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index 7e102e82..f12d8cbc 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -8,15 +8,24 @@ use tokio::sync::RwLock; use tokio::time::timeout; use tracing::debug; -/// copy of quic-proxy AutoReconnect - used that for reference +/// specialized connection manager with automatic reconnect; designated for connection lite-rpc -> quic proxy +/// +/// assumptions: +/// * connection to proxy service is reliable +/// * connection to proxy service is fast (50ms-200ms) and high-bandwidth +/// * proxy service will eventually be up again +/// * proxy location/IP address might change transparently +/// * proxy might be operated behind a load-balancer +/// * proxy will propagate backpressure by closing the connection +/// * proxy will not stall/delay connection but will respond fast with an error signal const SEND_TIMEOUT: Duration = Duration::from_secs(5); enum ConnectionState { NotConnected, Connection(Connection), - // not used for ocnnection to proxy - PermanentError, + // not used for connection to proxy + _PermanentError, FailedAttempt(u32), } @@ -38,7 +47,7 @@ impl AutoReconnect { pub async fn is_permanent_dead(&self) -> bool { let lock = self.current.read().await; - matches!(&*lock, ConnectionState::PermanentError) + matches!(&*lock, ConnectionState::_PermanentError) } pub async fn send_uni(&self, payload: &Vec) -> anyhow::Result<()> { @@ -58,7 +67,7 @@ impl AutoReconnect { match &*lock { ConnectionState::NotConnected => bail!("not connected"), ConnectionState::Connection(conn) => Ok(conn.clone()), - ConnectionState::PermanentError => bail!("permanent error"), + ConnectionState::_PermanentError => bail!("permanent error"), ConnectionState::FailedAttempt(_) => bail!("failed connection attempt"), } } @@ -133,7 +142,7 @@ impl AutoReconnect { } }; } - ConnectionState::PermanentError => { + ConnectionState::_PermanentError => { // no nothing debug!( "Not using connection to {} with permanent error", @@ -193,7 +202,7 @@ impl AutoReconnect { conn.stats().path.rtt ), ConnectionState::NotConnected => "n/c".to_string(), - ConnectionState::PermanentError => "n/a (permanent)".to_string(), + ConnectionState::_PermanentError => "n/a (permanent)".to_string(), ConnectionState::FailedAttempt(_) => "fail".to_string(), } } From 1a85527f74244b3bb4addee8cfac8ddbbdaf83ec Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 10:54:48 +0200 Subject: [PATCH 33/42] code fmt --- bench/src/bench_direct_quic.rs | 21 ++++++------ .../src/inbound/proxy_listener.rs | 34 +++++++++---------- quic-forward-proxy/src/main.rs | 2 +- quic-forward-proxy/src/outbound/tx_forward.rs | 7 ++-- services/src/tpu_utils/tpu_service.rs | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index d978ff59..f344cd7f 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -18,21 +18,19 @@ use solana_rpc_client::rpc_client::SerializableTransaction; use solana_sdk::signature::Signature; use solana_sdk::transaction::Transaction; use solana_sdk::{ - commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, - signer::Signer, slot_history::Slot, + commitment_config::CommitmentConfig, hash::Hash, signature::Keypair, signer::Signer, + slot_history::Slot, }; use std::fs; use std::fs::read_to_string; use std::net::{SocketAddr, SocketAddrV4}; use std::str::FromStr; use std::sync::atomic::AtomicBool; -use std::time::SystemTime; -use std::{ - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, }; +use std::time::SystemTime; use tokio::{ sync::{mpsc::UnboundedSender, RwLock}, time::{Duration, Instant}, @@ -208,8 +206,11 @@ async fn bench( ); for tpu_address in &leader_addrs { let tx_raw = bincode::serialize::(&tx).unwrap(); - let packet = - ForwardPacket::new(vec![tx_raw], SocketAddr::from(*tpu_address), 0xdeadbeef); + let packet = ForwardPacket::new( + vec![tx_raw], + SocketAddr::from(*tpu_address), + 0xdeadbeef, + ); forwarder_channel.send(packet).await.unwrap(); diff --git a/quic-forward-proxy/src/inbound/proxy_listener.rs b/quic-forward-proxy/src/inbound/proxy_listener.rs index 0a370e3a..f98d30d9 100644 --- a/quic-forward-proxy/src/inbound/proxy_listener.rs +++ b/quic-forward-proxy/src/inbound/proxy_listener.rs @@ -4,7 +4,7 @@ use crate::shared::ForwardPacket; use crate::tls_config_provider_server::ProxyTlsConfigProvider; use crate::tls_self_signed_pair_generator::SelfSignedTlsConfigProvider; use crate::util::FALLBACK_TIMEOUT; -use anyhow::{anyhow, bail, Context}; +use anyhow::{bail, Context}; use log::{debug, error, info, trace, warn}; use quinn::{Connecting, Endpoint, ServerConfig, VarInt}; use solana_sdk::packet::PACKET_DATA_SIZE; @@ -134,22 +134,20 @@ impl ProxyListener { for tpu_node in proxy_request.get_tpu_nodes() { let tpu_address = tpu_node.tpu_socket_addr; - let send_result = - forwarder_channel_copy - .send_timeout( - ForwardPacket::new( - txs.clone(), - tpu_address, - proxy_request.get_hash(), - ), - FALLBACK_TIMEOUT, - ) - .await - .context("sending internal packet from proxy to forwarder"); + let send_result = forwarder_channel_copy + .send_timeout( + ForwardPacket::new( + txs.clone(), + tpu_address, + proxy_request.get_hash(), + ), + FALLBACK_TIMEOUT, + ) + .await + .context("sending internal packet from proxy to forwarder"); if let Err(err) = send_result { - error!("send failed: {}", - err); + error!("send failed: {}", err); return; } } @@ -163,8 +161,10 @@ impl ProxyListener { Err(quinn::ConnectionError::ApplicationClosed(reason)) => { debug!("connection closed by client - reason: {:?}", reason); if reason.error_code != VarInt::from_u32(0) { - bail!("connection closed by client with unexpected reason: {:?}", - reason); + bail!( + "connection closed by client with unexpected reason: {:?}", + reason + ); } debug!("connection gracefully closed by client"); return Ok(()); diff --git a/quic-forward-proxy/src/main.rs b/quic-forward-proxy/src/main.rs index 652f4315..1c7d96dd 100644 --- a/quic-forward-proxy/src/main.rs +++ b/quic-forward-proxy/src/main.rs @@ -6,8 +6,8 @@ use clap::Parser; use dotenv::dotenv; use log::info; use solana_lite_rpc_core::keypair_loader::load_identity_keypair; -use std::sync::Arc; use solana_lite_rpc_services::prometheus_sync::PrometheusSync; +use std::sync::Arc; use crate::validator_identity::ValidatorIdentity; diff --git a/quic-forward-proxy/src/outbound/tx_forward.rs b/quic-forward-proxy/src/outbound/tx_forward.rs index dd10c7f2..e4843d9b 100644 --- a/quic-forward-proxy/src/outbound/tx_forward.rs +++ b/quic-forward-proxy/src/outbound/tx_forward.rs @@ -8,6 +8,7 @@ use crate::validator_identity::ValidatorIdentity; use anyhow::{bail, Context}; use futures::future::join_all; use log::{debug, info, trace, warn}; +use prometheus::{opts, register_int_counter, register_int_gauge, IntCounter, IntGauge}; use quinn::{ ClientConfig, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, VarInt, }; @@ -19,7 +20,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; -use prometheus::{IntCounter, IntGauge, opts, register_int_counter, register_int_gauge}; use tokio::sync::mpsc::Receiver; use tokio::sync::RwLock; @@ -75,7 +75,7 @@ pub async fn tx_forwarder( transaction_channel .recv() .await - .ok_or(anyhow::anyhow!("transaction_channel closed"))? + .ok_or(anyhow::anyhow!("transaction_channel closed"))?, ); let tpu_address = forward_packet.tpu_address; @@ -226,8 +226,7 @@ pub async fn tx_forwarder( debug!("tx-forward queue len: {}", broadcast_in.len()) } - let enqueue_result = broadcast_in - .send(forward_packet); + let enqueue_result = broadcast_in.send(forward_packet); if let Err(e) = enqueue_result { warn!("broadcast channel send error: {}", e); diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 09155167..e792328a 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -18,11 +18,11 @@ use std::collections::HashMap; use std::fs::File; use std::io::Write; use std::net::SocketAddr; +use std::path::Path; use std::{ net::{IpAddr, Ipv4Addr}, sync::Arc, }; -use std::path::Path; use tokio::time::Duration; lazy_static::lazy_static! { From cfaf5f181f3ecbd78ad34b6d7f28a6a5c38c416c Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:07:58 +0200 Subject: [PATCH 34/42] add bench-direct-quic to docker build --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5c826556..e57bc48c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,13 @@ FROM base as build COPY --from=plan /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release --bin lite-rpc --bin solana-lite-rpc-quic-forward-proxy --bin bench +RUN cargo build --release --bin lite-rpc --bin solana-lite-rpc-quic-forward-proxy --bin bench --bin bench-direct-quic FROM debian:bullseye-slim as run RUN apt-get update && apt-get -y install ca-certificates libc6 COPY --from=build /app/target/release/solana-lite-rpc-quic-forward-proxy /usr/local/bin/ COPY --from=build /app/target/release/lite-rpc /usr/local/bin/ COPY --from=build /app/target/release/bench /usr/local/bin/ +COPY --from=build /app/target/release/bench-direct-quic /usr/local/bin/ CMD lite-rpc --rpc-addr "$RPC_URL" --ws-addr "$WS_URL" From 0ab364a171dacf8bea09b6674666d2a20e3ab323 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:09:09 +0200 Subject: [PATCH 35/42] rename bench to bench_rpc --- Dockerfile | 4 ++-- bench/Cargo.toml | 4 ++++ bench/src/{main.rs => bench_rpc.rs} | 0 3 files changed, 6 insertions(+), 2 deletions(-) rename bench/src/{main.rs => bench_rpc.rs} (100%) diff --git a/Dockerfile b/Dockerfile index e57bc48c..4c856301 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,13 +14,13 @@ FROM base as build COPY --from=plan /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json COPY . . -RUN cargo build --release --bin lite-rpc --bin solana-lite-rpc-quic-forward-proxy --bin bench --bin bench-direct-quic +RUN cargo build --release --bin lite-rpc --bin solana-lite-rpc-quic-forward-proxy --bin bench-rpc --bin bench-direct-quic FROM debian:bullseye-slim as run RUN apt-get update && apt-get -y install ca-certificates libc6 COPY --from=build /app/target/release/solana-lite-rpc-quic-forward-proxy /usr/local/bin/ COPY --from=build /app/target/release/lite-rpc /usr/local/bin/ -COPY --from=build /app/target/release/bench /usr/local/bin/ +COPY --from=build /app/target/release/bench-rpc /usr/local/bin/ COPY --from=build /app/target/release/bench-direct-quic /usr/local/bin/ CMD lite-rpc --rpc-addr "$RPC_URL" --ws-addr "$WS_URL" diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 46cf680e..7ecaf9eb 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -3,6 +3,10 @@ name = "bench" version = "0.2.3" edition = "2021" +[[bin]] +name = "bench-rpc" +path = "src/bench_rpc.rs" + [[bin]] name = "bench-direct-quic" path = "src/bench_direct_quic.rs" diff --git a/bench/src/main.rs b/bench/src/bench_rpc.rs similarity index 100% rename from bench/src/main.rs rename to bench/src/bench_rpc.rs From 176cc868db9e027a8845fa6506d8500c1bccf1ef Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:10:13 +0200 Subject: [PATCH 36/42] remove unused dependencies --- bench/Cargo.toml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 7ecaf9eb..cb42ef62 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -15,6 +15,7 @@ path = "src/bench_direct_quic.rs" solana-lite-rpc-quic-forward-proxy = { path = "../quic-forward-proxy" } solana-sdk = { workspace = true } solana-rpc-client = { workspace = true } +solana-streamer = {workspace = true} log = { workspace = true } anyhow = { workspace = true } serde = { workspace = true } @@ -30,7 +31,4 @@ rand_chacha = "0.3.1" futures = { workspace = true } dashmap = { workspace = true } lazy_static = "1.4.0" -solana-lite-rpc-core = {workspace = true} -solana-lite-rpc-services = {workspace = true} -solana-streamer = {workspace = true} From 163e649c2293ff2fad064157764d53e91f1c4195 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:15:31 +0200 Subject: [PATCH 37/42] refactor bench files --- Cargo.lock | 2 -- bench/src/bench_direct_quic.rs | 9 +++++---- bench/src/bench_rpc.rs | 16 +++++++++------- bench/src/{cli.rs => cli_rpc.rs} | 0 bench/src/lib.rs | 3 --- 5 files changed, 14 insertions(+), 16 deletions(-) rename bench/src/{cli.rs => cli_rpc.rs} (100%) delete mode 100644 bench/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 73c3b4c0..e2608d80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -586,9 +586,7 @@ dependencies = [ "rand_chacha 0.3.1", "serde", "serde_json", - "solana-lite-rpc-core", "solana-lite-rpc-quic-forward-proxy", - "solana-lite-rpc-services", "solana-rpc-client", "solana-sdk", "solana-streamer", diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index f344cd7f..b42aa763 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -1,10 +1,11 @@ mod cli_direct_quic; +mod helpers; +mod metrics; use anyhow::Context; -use bench::{ - helpers::BenchHelper, - metrics::{AvgMetric, Metric, TxMetricData}, -}; +use helpers::BenchHelper; +use metrics::{AvgMetric, Metric, TxMetricData}; + use clap::Parser; use dashmap::DashMap; use futures::future::join_all; diff --git a/bench/src/bench_rpc.rs b/bench/src/bench_rpc.rs index 3b278bda..b2053e3e 100644 --- a/bench/src/bench_rpc.rs +++ b/bench/src/bench_rpc.rs @@ -1,8 +1,10 @@ -use bench::{ - cli::Args, - helpers::BenchHelper, - metrics::{AvgMetric, Metric, TxMetricData}, -}; +mod cli_rpc; +mod helpers; +mod metrics; + +use helpers::BenchHelper; +use metrics::{AvgMetric, Metric, TxMetricData}; + use clap::Parser; use dashmap::DashMap; use futures::future::join_all; @@ -25,14 +27,14 @@ use tokio::{ async fn main() { tracing_subscriber::fmt::init(); - let Args { + let cli_rpc::Args { tx_count, runs, run_interval_ms, metrics_file_name, lite_rpc_addr, transaction_save_file, - } = Args::parse(); + } = cli_rpc::Args::parse(); let mut run_interval_ms = tokio::time::interval(Duration::from_millis(run_interval_ms)); diff --git a/bench/src/cli.rs b/bench/src/cli_rpc.rs similarity index 100% rename from bench/src/cli.rs rename to bench/src/cli_rpc.rs diff --git a/bench/src/lib.rs b/bench/src/lib.rs deleted file mode 100644 index 2733319e..00000000 --- a/bench/src/lib.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod cli; -pub mod helpers; -pub mod metrics; From 7ab5ea02d2157447858ace950db86bb846a6caa6 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:15:42 +0200 Subject: [PATCH 38/42] add rust-toolchain.toml with rust 1.70.0 --- rust-toolchain.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 rust-toolchain.toml diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 00000000..22048ac5 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "1.70.0" From 001f61edd23054a98aa201a23f4fa2d7b6d5259e Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:31:13 +0200 Subject: [PATCH 39/42] refactor autoconnect --- bench/src/bench_direct_quic.rs | 3 ++ .../src/tpu_utils/quinn_auto_reconnect.rs | 40 ++++++++++--------- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index b42aa763..99735a92 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -1,3 +1,6 @@ +/// bench tool that uses the TPU client from quic proxy to submit transactions to TPUs via QUIC +/// +/// note: this tool requires a lite-rpc service that is configured to dump the current leader list to a file (leaders.dat) mod cli_direct_quic; mod helpers; mod metrics; diff --git a/services/src/tpu_utils/quinn_auto_reconnect.rs b/services/src/tpu_utils/quinn_auto_reconnect.rs index f12d8cbc..a1011234 100644 --- a/services/src/tpu_utils/quinn_auto_reconnect.rs +++ b/services/src/tpu_utils/quinn_auto_reconnect.rs @@ -30,7 +30,7 @@ enum ConnectionState { } pub struct AutoReconnect { - // endoint should be configures with keep-alive and idle timeout + // endpoint should be configures with keep-alive and idle timeout endpoint: Endpoint, current: RwLock, pub target_address: SocketAddr, @@ -45,7 +45,7 @@ impl AutoReconnect { } } - pub async fn is_permanent_dead(&self) -> bool { + pub async fn _is_permanent_dead(&self) -> bool { let lock = self.current.read().await; matches!(&*lock, ConnectionState::_PermanentError) } @@ -100,7 +100,7 @@ impl AutoReconnect { ); match self.create_connection().await { - Some(new_connection) => { + Ok(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); info!( "Restored closed connection {} with {} to target {}", @@ -109,10 +109,10 @@ impl AutoReconnect { self.target_address, ); } - None => { + Err(err) => { warn!( - "Reconnect to {} failed for connection {}", - self.target_address, old_stable_id + "Reconnect to {} failed for connection {}: {}", + self.target_address, old_stable_id, err ); *lock = ConnectionState::FailedAttempt(1); } @@ -127,7 +127,7 @@ impl AutoReconnect { } ConnectionState::NotConnected => { match self.create_connection().await { - Some(new_connection) => { + Ok(new_connection) => { *lock = ConnectionState::Connection(new_connection.clone()); info!( @@ -136,8 +136,11 @@ impl AutoReconnect { self.target_address ); } - None => { - warn!("Failed connect initially to target {}", self.target_address); + Err(err) => { + warn!( + "Failed connect initially to target {}: {}", + self.target_address, err + ); *lock = ConnectionState::FailedAttempt(1); } }; @@ -151,13 +154,13 @@ impl AutoReconnect { } ConnectionState::FailedAttempt(attempts) => { match self.create_connection().await { - Some(new_connection) => { + Ok(new_connection) => { *lock = ConnectionState::Connection(new_connection); } - None => { + Err(err) => { warn!( - "Reconnect to {} failed (attempt {})", - self.target_address, attempts + "Reconnect to {} failed (attempt {}): {}", + self.target_address, attempts, err ); *lock = ConnectionState::FailedAttempt(attempts + 1); } @@ -166,21 +169,22 @@ impl AutoReconnect { } } - async fn create_connection(&self) -> Option { + async fn create_connection(&self) -> anyhow::Result { let connection = self .endpoint .connect(self.target_address, "localhost") - .expect("handshake"); + .context("handshake")?; match connection.await { - Ok(conn) => Some(conn), - Err(ConnectionError::TimedOut) => None, + Ok(conn) => Ok(conn), + Err(ConnectionError::TimedOut) => bail!("timeout"), // maybe we should also treat TransportError explicitly Err(unexpected_error) => { - panic!( + warn!( "Connection to {} failed with unexpected error: {}", self.target_address, unexpected_error ); + bail!("Unecpected error connecting"); } } } From c7cc774f293f1eddec411a54f52bbd2af9e67716 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:31:32 +0200 Subject: [PATCH 40/42] fix minor rust issue --- bench/src/bench_direct_quic.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index 99735a92..ffd16d28 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -244,7 +244,7 @@ async fn bench( } let chunks = signatures.chunks(100).collect::>(); for chunk in chunks { - if let Ok(res) = rpc_client.get_signature_statuses(&chunk).await { + if let Ok(res) = rpc_client.get_signature_statuses(chunk).await { for (i, signature) in chunk.iter().enumerate() { let tx_status = &res.value[i]; if tx_status.is_some() { From 488ab746c2aab10ac906af89cc05438acc0855ac Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Tue, 19 Sep 2023 11:47:00 +0200 Subject: [PATCH 41/42] disable flaky with_1000_transactions_direct test --- .../tests/quic_proxy_tpu_integrationtest.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs index 50d249cd..0a563c79 100644 --- a/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs +++ b/quic-forward-proxy-integration-test/tests/quic_proxy_tpu_integrationtest.rs @@ -105,6 +105,7 @@ pub fn with_100_transactions_direct() { }); } +#[ignore] #[test] pub fn with_1000_transactions_direct() { configure_logging(false); From b343c1c0e8beae4543733c9dbaefc9c0df6ae8f5 Mon Sep 17 00:00:00 2001 From: GroovieGermanikus Date: Thu, 21 Sep 2023 14:45:58 +0200 Subject: [PATCH 42/42] add flag ENABLE_LEADERSDAT_FOR_BENCH for leaders.dat --- bench/src/bench_direct_quic.rs | 1 + services/src/tpu_utils/tpu_service.rs | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/bench/src/bench_direct_quic.rs b/bench/src/bench_direct_quic.rs index e9eb9b01..a4bc8a33 100644 --- a/bench/src/bench_direct_quic.rs +++ b/bench/src/bench_direct_quic.rs @@ -1,6 +1,7 @@ /// bench tool that uses the TPU client from quic proxy to submit transactions to TPUs via QUIC /// /// note: this tool requires a lite-rpc service that is configured to dump the current leader list to a file (leaders.dat) +/// important: need to enable flag ENABLE_LEADERSDAT_FOR_BENCH in tpu_service.rs mod cli_direct_quic; mod helpers; mod metrics; diff --git a/services/src/tpu_utils/tpu_service.rs b/services/src/tpu_utils/tpu_service.rs index 72df5d48..83f70db2 100644 --- a/services/src/tpu_utils/tpu_service.rs +++ b/services/src/tpu_utils/tpu_service.rs @@ -24,6 +24,9 @@ use std::{ sync::Arc, }; +// enable unless it is needed for bench_direct_quic.rs for performance reasons +const ENABLE_LEADERSDAT_FOR_BENCH: bool = false; + lazy_static::lazy_static! { static ref NB_CLUSTER_NODES: GenericGauge = register_int_gauge!(opts!("literpc_nb_cluster_nodes", "Number of cluster nodes in saved")).unwrap(); @@ -148,7 +151,9 @@ impl TpuService { }) .collect(); - dump_leaders_to_file(connections_to_keep.values().collect_vec()); + if ENABLE_LEADERSDAT_FOR_BENCH { + dump_leaders_to_file(connections_to_keep.values().collect_vec()); + } match &self.connection_manager { DirectTpu {