From fc0183d73477fc24ee4b7c46b80efdd44d4b9cf1 Mon Sep 17 00:00:00 2001 From: Lijun Wang <83639177+lijunwangs@users.noreply.github.com> Date: Sat, 14 Sep 2024 01:41:16 -0700 Subject: [PATCH] Use quinn 0.11.x (#1641) 1. A incoming connection can be ignored now when we do rate limiting -- which is more efficient -- does not require queueing outgoing packets. 2. rustls interface changes of ServerCertVerifier applied to SkipServerVerification implementations changes in Cargo.toml to handle curve25519-dalek 3.2.1 patching because of zeroize version constraint due to the newer rustls. The workaround is applied to downstream tests. 3. Quinn 0.11.x introduced new error codes which we need to handle. 4. stream finish is no longer an async function. --- .../scripts/downstream-project-spl-common.sh | 4 + Cargo.lock | 214 ++++++++++++++---- Cargo.toml | 8 +- core/src/repair/quic_endpoint.rs | 116 ++++++---- programs/sbf/Cargo.lock | 209 +++++++++++++---- quic-client/Cargo.toml | 2 +- quic-client/src/nonblocking/quic_client.rs | 69 ++++-- scripts/build-downstream-anchor-projects.sh | 1 + streamer/Cargo.toml | 2 +- streamer/src/nonblocking/quic.rs | 81 ++++--- streamer/src/nonblocking/testing_utilities.rs | 65 ++++-- streamer/src/quic.rs | 89 ++++++-- streamer/src/tls_certificates.rs | 15 +- turbine/src/quic_endpoint.rs | 91 +++++--- 14 files changed, 708 insertions(+), 258 deletions(-) diff --git a/.github/scripts/downstream-project-spl-common.sh b/.github/scripts/downstream-project-spl-common.sh index e70c10d070b7e9..f5e0a7e448f729 100644 --- a/.github/scripts/downstream-project-spl-common.sh +++ b/.github/scripts/downstream-project-spl-common.sh @@ -30,3 +30,7 @@ sed -i 's/solana-geyser-plugin-interface/agave-geyser-plugin-interface/g' ./Carg # should be removed when spl bump their curve25519-dalek sed -i "s/^curve25519-dalek =.*/curve25519-dalek = \"4.1.3\"/" token/confidential-transfer/proof-generation/Cargo.toml + +# fix curve25519-dalek + +sed -i '/\[patch.crates-io\]/a curve25519-dalek = { git = "https://github.com/anza-xyz/curve25519-dalek.git", rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" }' ./Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index fb0f87b78b7640..0ce95129d77bea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,7 +886,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.77", ] @@ -1279,6 +1279,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1470,6 +1476,16 @@ dependencies = [ "unreachable", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1749,9 +1765,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "3.2.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder", "digest 0.9.0", @@ -2043,7 +2059,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ - "curve25519-dalek 3.2.1", + "curve25519-dalek 3.2.0", "ed25519", "rand 0.7.3", "serde", @@ -2877,7 +2893,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", ] @@ -3087,6 +3103,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine 4.6.7", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.24" @@ -3847,9 +3883,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -4466,16 +4502,17 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.10.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2 0.5.7", "thiserror", "tokio", "tracing", @@ -4483,16 +4520,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand 0.8.5", - "ring 0.16.20", - "rustc-hash", - "rustls", - "rustls-native-certs", + "ring 0.17.3", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "rustls-platform-verifier", "slab", "thiserror", "tinyvec", @@ -4501,15 +4538,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.4.0" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df19e284d93757a9fb91d63672f7741b129246a669db09d1c0063071debc0c0" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ - "bytes", "libc", + "once_cell", "socket2 0.5.7", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4800,7 +4837,7 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite", - "rustls", + "rustls 0.21.12", "rustls-pemfile 1.0.0", "serde", "serde_json", @@ -4916,6 +4953,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.3.3" @@ -4964,40 +5007,89 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring 0.17.3", + "rustls-pki-types", + "rustls-webpki 0.102.7", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile 0.2.1", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "0.2.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" dependencies = [ "base64 0.13.1", ] [[package]] name = "rustls-pemfile" -version = "1.0.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-platform-verifier" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034f511ae2823d6a657925d5f7cf821d7458367ce0a54fbe743c471fdddd8111" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.102.7", + "security-framework", + "security-framework-sys", + "webpki-roots 0.26.5", + "winapi 0.3.9", ] +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5008,6 +5100,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +dependencies = [ + "ring 0.17.3", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -5081,22 +5184,23 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", + "num-bigint 0.4.6", "security-framework-sys", ] [[package]] name = "security-framework-sys" -version = "2.4.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -5442,9 +5546,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -6250,7 +6357,7 @@ dependencies = [ "rayon", "rolling-file", "rustc_version 0.4.1", - "rustls", + "rustls 0.23.12", "serde", "serde_bytes", "serde_derive", @@ -7298,7 +7405,7 @@ dependencies = [ "log", "quinn", "quinn-proto", - "rustls", + "rustls 0.23.12", "solana-connection-cache", "solana-logger", "solana-measure", @@ -7902,7 +8009,7 @@ dependencies = [ "quinn", "quinn-proto", "rand 0.8.5", - "rustls", + "rustls 0.23.12", "smallvec", "socket2 0.5.7", "solana-logger", @@ -8249,7 +8356,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rayon", - "rustls", + "rustls 0.23.12", "solana-entry", "solana-feature-set", "solana-gossip", @@ -8546,7 +8653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c1941b5ef0c3ce8f2ac5dd984d0fb1a97423c4ff2a02eec81e3913f02e2ac2b" dependencies = [ "byteorder", - "combine", + "combine 3.8.1", "gdbstub", "hash32", "libc", @@ -9194,7 +9301,7 @@ dependencies = [ "once_cell", "pbkdf2 0.4.0", "rand 0.7.3", - "rustc-hash", + "rustc-hash 1.1.0", "sha2 0.9.9", "thiserror", "unicode-normalization", @@ -9282,7 +9389,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", "tokio", ] @@ -9321,7 +9428,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", "tungstenite", @@ -9568,7 +9675,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls", + "rustls 0.21.12", "sha1", "thiserror", "url 2.5.2", @@ -9878,7 +9985,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -9887,6 +9994,15 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "webpki-roots" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.2.2" @@ -10215,9 +10331,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 2b8cbd137f4e12..6b4eac9eff9fe8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -325,8 +325,8 @@ prost-types = "0.11.9" protobuf-src = "1.1.0" qstring = "0.7.2" qualifier_attr = { version = "0.2.2", default-features = false } -quinn = "0.10.2" -quinn-proto = "0.10.6" +quinn = "0.11.4" +quinn-proto = "0.11.7" quote = "1.0" rand = "0.8.5" rand_chacha = "0.3.1" @@ -338,7 +338,7 @@ reqwest-middleware = "0.2.5" rolling-file = "0.2.0" rpassword = "7.3" rustc_version = "0.4" -rustls = { version = "0.21.12", default-features = false, features = ["quic"] } +rustls = { version = "0.23.9", default-features = false } scopeguard = "1.2.0" semver = "1.0.23" seqlock = "0.2.0" @@ -522,7 +522,7 @@ winapi = "0.3.8" winreg = "0.50" x509-parser = "0.14.0" # See "zeroize versioning issues" below if you are updating this version. -zeroize = { version = "1.3", default-features = false } +zeroize = { version = "1.7", default-features = false } zstd = "0.13.2" [patch.crates-io] diff --git a/core/src/repair/quic_endpoint.rs b/core/src/repair/quic_endpoint.rs index abaf6d03483f10..5f4fea3f637e73 100644 --- a/core/src/repair/quic_endpoint.rs +++ b/core/src/repair/quic_endpoint.rs @@ -5,11 +5,15 @@ use { itertools::Itertools, log::error, quinn::{ - ClientConfig, ConnectError, Connecting, Connection, ConnectionError, Endpoint, - EndpointConfig, IdleTimeout, ReadError, ReadToEndError, RecvStream, SendStream, + crypto::rustls::{QuicClientConfig, QuicServerConfig}, + ClientConfig, ClosedStream, ConnectError, Connecting, Connection, ConnectionError, + Endpoint, EndpointConfig, IdleTimeout, ReadError, ReadToEndError, RecvStream, SendStream, ServerConfig, TokioRuntime, TransportConfig, VarInt, WriteError, }, - rustls::{Certificate, KeyLogFile, PrivateKey}, + rustls::{ + pki_types::{CertificateDer, PrivateKeyDer}, + CertificateError, KeyLogFile, + }, serde_bytes::ByteBuf, solana_quic_client::nonblocking::quic_client::SkipServerVerification, solana_runtime::bank_forks::BankForks, @@ -105,6 +109,8 @@ pub(crate) enum Error { TlsError(#[from] rustls::Error), #[error(transparent)] WriteError(#[from] WriteError), + #[error(transparent)] + ClosedStream(#[from] ClosedStream), } macro_rules! add_metric { @@ -122,7 +128,7 @@ pub(crate) fn new_quic_endpoint( bank_forks: Arc>, ) -> Result<(Endpoint, AsyncSender, AsyncTryJoinHandle), Error> { let (cert, key) = new_dummy_x509_certificate(keypair); - let server_config = new_server_config(cert.clone(), key.clone())?; + let server_config = new_server_config(cert.clone(), key.clone_key())?; let client_config = new_client_config(cert, key)?; let mut endpoint = { // Endpoint::new requires entering the runtime context, @@ -168,29 +174,36 @@ pub(crate) fn close_quic_endpoint(endpoint: &Endpoint) { ); } -fn new_server_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_server_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(SkipClientVerification {})) + .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], key)?; config.alpn_protocols = vec![ALPN_REPAIR_PROTOCOL_ID.to_vec()]; config.key_log = Arc::new(KeyLogFile::new()); - let mut config = ServerConfig::with_crypto(Arc::new(config)); + let quic_server_config = QuicServerConfig::try_from(config) + .map_err(|_err| rustls::Error::InvalidCertificate(CertificateError::BadSignature))?; + + let mut config = ServerConfig::with_crypto(Arc::new(quic_server_config)); config .transport_config(Arc::new(new_transport_config())) - .use_retry(true) .migration(false); Ok(config) } -fn new_client_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_client_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(Arc::new(SkipServerVerification {})) + .dangerous() + .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key)?; config.enable_early_data = true; config.alpn_protocols = vec![ALPN_REPAIR_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(config)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(config).unwrap())); config.transport_config(Arc::new(new_transport_config())); Ok(config) } @@ -219,17 +232,26 @@ async fn run_server( let stats = Arc::::default(); let report_metrics_task = tokio::task::spawn(report_metrics_task("repair_quic_server", stats.clone())); - while let Some(connecting) = endpoint.accept().await { - tokio::task::spawn(handle_connecting_task( - endpoint.clone(), - connecting, - remote_request_sender.clone(), - bank_forks.clone(), - prune_cache_pending.clone(), - router.clone(), - cache.clone(), - stats.clone(), - )); + while let Some(incoming) = endpoint.accept().await { + let remote_addr: SocketAddr = incoming.remote_address(); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::task::spawn(handle_connecting_task( + endpoint.clone(), + connecting, + remote_request_sender.clone(), + bank_forks.clone(), + prune_cache_pending.clone(), + router.clone(), + cache.clone(), + stats.clone(), + )); + } + Err(error) => { + debug!("Error while accepting incoming connection: {error:?} from {remote_addr}"); + } + } } report_metrics_task.abort(); } @@ -504,7 +526,7 @@ async fn handle_streams( send_stream.write_all(&size.to_le_bytes()).await?; send_stream.write_all(&chunk).await?; } - send_stream.finish().await.map_err(Error::from) + send_stream.finish().map_err(Error::from) } async fn send_requests_task( @@ -565,7 +587,7 @@ async fn send_request( const READ_TIMEOUT_DURATION: Duration = Duration::from_secs(10); let (mut send_stream, mut recv_stream) = connection.open_bi().await?; send_stream.write_all(&bytes).await?; - send_stream.finish().await?; + send_stream.finish()?; // Each response is at most PACKET_DATA_SIZE bytes and requires // an additional 8 bytes to encode its length. let size = PACKET_DATA_SIZE @@ -778,6 +800,7 @@ struct RepairQuicStats { connection_error_timed_out: AtomicU64, connection_error_transport_error: AtomicU64, connection_error_version_mismatch: AtomicU64, + connection_error_connection_limit_exceeded: AtomicU64, invalid_identity: AtomicU64, no_response_received: AtomicU64, read_to_end_error_connection_lost: AtomicU64, @@ -792,6 +815,12 @@ struct RepairQuicStats { write_error_stopped: AtomicU64, write_error_unknown_stream: AtomicU64, write_error_zero_rtt_rejected: AtomicU64, + connect_error_cids_exhausted: AtomicU64, + connect_error_invalid_server_name: AtomicU64, + connection_error_cids_exhausted: AtomicU64, + closed_streams: AtomicU64, + read_to_end_error_closed_stream: AtomicU64, + write_error_closed_stream: AtomicU64, } async fn report_metrics_task(name: &'static str, stats: Arc) { @@ -808,12 +837,6 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::ConnectError(ConnectError::EndpointStopping) => { add_metric!(stats.connect_error_other) } - Error::ConnectError(ConnectError::TooManyConnections) => { - add_metric!(stats.connect_error_too_many_connections) - } - Error::ConnectError(ConnectError::InvalidDnsName(_)) => { - add_metric!(stats.connect_error_other) - } Error::ConnectError(ConnectError::InvalidRemoteAddress(_)) => { add_metric!(stats.connect_error_invalid_remote_address) } @@ -851,9 +874,6 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::ReadToEndError(ReadToEndError::Read(ReadError::ConnectionLost(_))) => { add_metric!(stats.read_to_end_error_connection_lost) } - Error::ReadToEndError(ReadToEndError::Read(ReadError::UnknownStream)) => { - add_metric!(stats.read_to_end_error_unknown_stream) - } Error::ReadToEndError(ReadToEndError::Read(ReadError::IllegalOrderedRead)) => { add_metric!(stats.read_to_end_error_illegal_ordered_read) } @@ -869,12 +889,27 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::WriteError(WriteError::ConnectionLost(_)) => { add_metric!(stats.write_error_connection_lost) } - Error::WriteError(WriteError::UnknownStream) => { - add_metric!(stats.write_error_unknown_stream) - } Error::WriteError(WriteError::ZeroRttRejected) => { add_metric!(stats.write_error_zero_rtt_rejected) } + Error::ConnectError(ConnectError::CidsExhausted) => { + add_metric!(stats.connect_error_cids_exhausted) + } + Error::ConnectError(ConnectError::InvalidServerName(_)) => { + add_metric!(stats.connect_error_invalid_server_name) + } + Error::ConnectionError(ConnectionError::CidsExhausted) => { + add_metric!(stats.connection_error_cids_exhausted) + } + Error::ClosedStream(_) => { + add_metric!(stats.closed_streams) + } + Error::ReadToEndError(ReadToEndError::Read(ReadError::ClosedStream)) => { + add_metric!(stats.read_to_end_error_closed_stream) + } + Error::WriteError(WriteError::ClosedStream) => { + add_metric!(stats.write_error_closed_stream) + } } } @@ -936,6 +971,11 @@ fn report_metrics(name: &'static str, stats: &RepairQuicStats) { reset_metric!(stats.connection_error_version_mismatch), i64 ), + ( + "connection_error_connection_limit_exceeded", + reset_metric!(stats.connection_error_connection_limit_exceeded), + i64 + ), ( "invalid_identity", reset_metric!(stats.invalid_identity), diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cbe090c6449fe3..c275a7fe049f48 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -662,7 +662,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.58", ] @@ -927,6 +927,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1027,6 +1033,16 @@ dependencies = [ "unreachable", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1207,9 +1223,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "3.2.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder 1.5.0", "digest 0.9.0", @@ -1484,7 +1500,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ - "curve25519-dalek 3.2.1", + "curve25519-dalek 3.2.0", "ed25519", "rand 0.7.3", "serde", @@ -2209,7 +2225,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", ] @@ -2408,6 +2424,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine 4.6.7", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.21" @@ -3208,9 +3244,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -3739,16 +3775,17 @@ dependencies = [ [[package]] name = "quinn" -version = "0.10.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2 0.5.7", "thiserror", "tokio", "tracing", @@ -3756,16 +3793,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand 0.8.5", - "ring 0.16.20", - "rustc-hash", - "rustls", - "rustls-native-certs", + "ring 0.17.3", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "rustls-platform-verifier", "slab", "thiserror", "tinyvec", @@ -3774,15 +3811,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.4.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ - "bytes", "libc", + "once_cell", "socket2 0.5.7", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4008,8 +4045,8 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.0", "serde", "serde_json", "serde_urlencoded", @@ -4124,6 +4161,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -4163,18 +4206,33 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring 0.17.3", + "rustls-pki-types", + "rustls-webpki 0.102.7", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] @@ -4188,6 +4246,49 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-platform-verifier" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbb878bdfdf63a336a5e63561b1835e7a8c91524f51621db870169eac84b490" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.102.7", + "security-framework", + "security-framework-sys", + "webpki-roots 0.26.5", + "winapi 0.3.9", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4198,6 +4299,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +dependencies = [ + "ring 0.17.3", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -4260,6 +4372,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "libc", + "num-bigint 0.4.6", "security-framework-sys", ] @@ -4504,9 +4617,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -4959,7 +5075,7 @@ dependencies = [ "rand_chacha 0.3.1", "rayon", "rolling-file", - "rustls", + "rustls 0.23.12", "serde", "serde_bytes", "serde_derive", @@ -5661,7 +5777,7 @@ dependencies = [ "log", "quinn", "quinn-proto", - "rustls", + "rustls 0.23.12", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -6603,7 +6719,7 @@ dependencies = [ "quinn", "quinn-proto", "rand 0.8.5", - "rustls", + "rustls 0.23.12", "smallvec", "socket2 0.5.7", "solana-measure", @@ -6819,7 +6935,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rayon", - "rustls", + "rustls 0.23.12", "solana-entry", "solana-feature-set", "solana-gossip", @@ -7047,7 +7163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c1941b5ef0c3ce8f2ac5dd984d0fb1a97423c4ff2a02eec81e3913f02e2ac2b" dependencies = [ "byteorder 1.5.0", - "combine", + "combine 3.8.1", "hash32", "libc", "log", @@ -7630,7 +7746,7 @@ dependencies = [ "once_cell", "pbkdf2 0.4.0", "rand 0.7.3", - "rustc-hash", + "rustc-hash 1.1.0", "sha2 0.9.9", "thiserror", "unicode-normalization", @@ -7710,7 +7826,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", "tokio", ] @@ -7749,7 +7865,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", "tungstenite", @@ -7833,7 +7949,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project", "prost", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "tokio", "tokio-rustls", "tokio-stream", @@ -7971,7 +8087,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls", + "rustls 0.21.12", "sha1", "thiserror", "url 2.5.2", @@ -8260,7 +8376,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -8269,6 +8385,15 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "webpki-roots" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.2.5" @@ -8525,9 +8650,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] diff --git a/quic-client/Cargo.toml b/quic-client/Cargo.toml index 05800896c30f80..428b03cb0a6892 100644 --- a/quic-client/Cargo.toml +++ b/quic-client/Cargo.toml @@ -18,7 +18,7 @@ lazy_static = { workspace = true } log = { workspace = true } quinn = { workspace = true } quinn-proto = { workspace = true } -rustls = { workspace = true, features = ["dangerous_configuration"] } +rustls = { workspace = true } solana-connection-cache = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/quic-client/src/nonblocking/quic_client.rs b/quic-client/src/nonblocking/quic_client.rs index b40e89ec216c9d..c6a66fe56bcd10 100644 --- a/quic-client/src/nonblocking/quic_client.rs +++ b/quic-client/src/nonblocking/quic_client.rs @@ -8,8 +8,9 @@ use { itertools::Itertools, log::*, quinn::{ - ClientConfig, ConnectError, Connection, ConnectionError, Endpoint, EndpointConfig, - IdleTimeout, TokioRuntime, TransportConfig, WriteError, + crypto::rustls::QuicClientConfig, ClientConfig, ClosedStream, ConnectError, Connection, + ConnectionError, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, + WriteError, }, solana_connection_cache::{ client_connection::ClientStats, connection_cache_stats::ConnectionCacheStats, @@ -38,31 +39,63 @@ use { tokio::{sync::OnceCell, time::timeout}, }; -pub struct SkipServerVerification; +#[derive(Debug)] +pub struct SkipServerVerification(Arc); impl SkipServerVerification { pub fn new() -> Arc { - Arc::new(Self) + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } -impl rustls::client::ServerCertVerifier for SkipServerVerification { +impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + fn verify_server_cert( &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) } } pub struct QuicClientCertificate { - pub certificate: rustls::Certificate, - pub key: rustls::PrivateKey, + pub certificate: rustls::pki_types::CertificateDer<'static>, + pub key: rustls::pki_types::PrivateKeyDer<'static>, } /// A lazy-initialized Quic Endpoint @@ -80,6 +113,8 @@ pub enum QuicError { ConnectionError(#[from] ConnectionError), #[error(transparent)] ConnectError(#[from] ConnectError), + #[error(transparent)] + ClosedStream(#[from] ClosedStream), } impl From for ClientErrorKind { @@ -115,17 +150,17 @@ impl QuicLazyInitializedEndpoint { }; let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() + .dangerous() .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert( vec![self.client_certificate.certificate.clone()], - self.client_certificate.key.clone(), + self.client_certificate.key.clone_key(), ) .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(crypto)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(crypto).unwrap())); let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); diff --git a/scripts/build-downstream-anchor-projects.sh b/scripts/build-downstream-anchor-projects.sh index e282a446c7c79a..fe2f5e813ee613 100755 --- a/scripts/build-downstream-anchor-projects.sh +++ b/scripts/build-downstream-anchor-projects.sh @@ -69,6 +69,7 @@ anchor() { patch_crates_io_solana Cargo.toml "$solana_dir" patch_spl_crates . Cargo.toml "$spl_dir" + sed -i '/\[patch.crates-io\]/a curve25519-dalek = { git = "https://github.com/anza-xyz/curve25519-dalek.git", rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" }' ./Cargo.toml $cargo test # serum_dex and mpl-token-metadata are using caret versions of solana and SPL dependencies # rather pull and patch those as well, ignore for now diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index 89ce80c910c80f..873cb4459e3327 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -28,7 +28,7 @@ percentage = { workspace = true } quinn = { workspace = true } quinn-proto = { workspace = true } rand = { workspace = true } -rustls = { workspace = true, features = ["dangerous_configuration"] } +rustls = { workspace = true } smallvec = { workspace = true } socket2 = { workspace = true } solana-measure = { workspace = true } diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index f1b0a5a7efd5ed..760902a6aab06b 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -198,7 +198,7 @@ pub fn spawn_server_multi( let concurrent_connections = (max_staked_connections + max_unstaked_connections) / sockets.len(); let max_concurrent_connections = concurrent_connections + concurrent_connections / 4; - let (config, _cert) = configure_server(keypair, max_concurrent_connections)?; + let (config, _) = configure_server(keypair)?; let endpoints = sockets .into_iter() @@ -239,7 +239,7 @@ pub fn spawn_server_multi( #[allow(clippy::too_many_arguments)] async fn run_server( name: &'static str, - incoming: Vec, + endpoints: Vec, packet_sender: Sender, exit: Arc, max_connections_per_peer: usize, @@ -268,7 +268,7 @@ async fn run_server( )); stats .quic_endpoints_count - .store(incoming.len(), Ordering::Relaxed); + .store(endpoints.len(), Ordering::Relaxed); let staked_connection_table: Arc> = Arc::new(Mutex::new(ConnectionTable::new())); let (sender, receiver) = async_unbounded(); @@ -280,7 +280,7 @@ async fn run_server( coalesce, )); - let mut accepts = incoming + let mut accepts = endpoints .iter() .enumerate() .map(|(i, incoming)| { @@ -296,7 +296,7 @@ async fn run_server( if let Some((connecting, i)) = ready { accepts.push( Box::pin(EndpointAccept { - accept: incoming[i].accept(), + accept: endpoints[i].accept(), endpoint: i, } )); @@ -316,11 +316,11 @@ async fn run_server( last_datapoint = Instant::now(); } - if let Ok(Some(connection)) = timeout_connection { + if let Ok(Some(incoming)) = timeout_connection { stats .total_incoming_connection_attempts .fetch_add(1, Ordering::Relaxed); - let remote_address = connection.remote_address(); + let remote_address = incoming.remote_address(); // first check overall connection rate limit: if !overall_connection_rate_limiter.is_allowed() { @@ -331,6 +331,7 @@ async fn run_server( stats .connection_rate_limited_across_all .fetch_add(1, Ordering::Relaxed); + incoming.ignore(); continue; } @@ -349,26 +350,35 @@ async fn run_server( stats .connection_rate_limited_per_ipaddr .fetch_add(1, Ordering::Relaxed); + incoming.ignore(); continue; } stats .outstanding_incoming_connection_attempts .fetch_add(1, Ordering::Relaxed); - tokio::spawn(setup_connection( - connection, - unstaked_connection_table.clone(), - staked_connection_table.clone(), - sender.clone(), - max_connections_per_peer, - staked_nodes.clone(), - max_staked_connections, - max_unstaked_connections, - max_streams_per_ms, - stats.clone(), - wait_for_chunk_timeout, - stream_load_ema.clone(), - )); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::spawn(setup_connection( + connecting, + unstaked_connection_table.clone(), + staked_connection_table.clone(), + sender.clone(), + max_connections_per_peer, + staked_nodes.clone(), + max_staked_connections, + max_unstaked_connections, + max_streams_per_ms, + stats.clone(), + wait_for_chunk_timeout, + stream_load_ema.clone(), + )); + } + Err(err) => { + debug!("Incoming::accept(): error {:?}", err); + } + } } else { debug!("accept(): Timed out waiting for connection"); } @@ -394,7 +404,7 @@ pub fn get_remote_pubkey(connection: &Connection) -> Option { // Use the client cert only if it is self signed and the chain length is 1. connection .peer_identity()? - .downcast::>() + .downcast::>() .ok() .filter(|certs| certs.len() == 1)? .first() @@ -1263,7 +1273,7 @@ impl Drop for ConnectionEntry { } } -#[derive(Copy, Clone, Eq, Hash, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] enum ConnectionTableKey { IP(IpAddr), Pubkey(Pubkey), @@ -1422,7 +1432,7 @@ struct EndpointAccept<'a> { } impl<'a> Future for EndpointAccept<'a> { - type Output = (Option, usize); + type Output = (Option, usize); fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context) -> Poll { let i = self.endpoint; @@ -1463,7 +1473,7 @@ pub mod test { for i in 0..total { let mut s1 = conn1.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); info!("done {}", i); sleep(Duration::from_millis(1000)).await; } @@ -1488,7 +1498,7 @@ pub mod test { let s2 = conn2.open_uni().await; if let Ok(mut s2) = s2 { s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); // Send enough data to create more than 1 chunks. // The first will try to open the connection (which should fail). // The following chunks will enable the detection of connection failure. @@ -1496,9 +1506,6 @@ pub mod test { s2.write_all(&data) .await .expect_err("shouldn't be able to open 2 connections"); - s2.finish() - .await - .expect_err("shouldn't be able to open 2 connections"); } else { // It has been noticed if there is already connection open against the server, this open_uni can fail // with ApplicationClosed(ApplicationClose) error due to CONNECTION_CLOSE_CODE_TOO_MANY before writing to @@ -1521,9 +1528,9 @@ pub mod test { let mut s1 = c1.open_uni().await.unwrap(); let mut s2 = c2.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); s2.write_all(&[0u8]).await.unwrap(); - s2.finish().await.unwrap(); + s2.finish().unwrap(); num_expected_packets += 2; sleep(Duration::from_millis(200)).await; } @@ -1563,7 +1570,7 @@ pub mod test { for _ in 0..num_bytes { s1.write_all(&[0u8]).await.unwrap(); } - s1.finish().await.unwrap(); + s1.finish().unwrap(); let mut all_packets = vec![]; let now = Instant::now(); @@ -1598,7 +1605,8 @@ pub mod test { // Ignoring any errors here. s1.finish() will test the error condition s1.write_all(&[0u8]).await.unwrap_or_default(); } - s1.finish().await.unwrap_err(); + s1.finish().unwrap_or_default(); + s1.stopped().await.unwrap_err(); } } @@ -1712,7 +1720,6 @@ pub mod test { // Test that more writes to the stream will fail (i.e. the stream is no longer writable // after the timeouts) assert!(s1.write_all(&[0u8]).await.is_err()); - assert!(s1.finish().await.is_err()); exit.store(true, Ordering::Relaxed); join_handle.await.unwrap(); @@ -1775,7 +1782,7 @@ pub mod test { let mut s1 = conn1.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); let mut s2 = conn2.open_uni().await.unwrap(); conn1.close( @@ -1789,7 +1796,7 @@ pub mod test { assert_eq!(stats.connection_removed.load(Ordering::Relaxed), 1); s2.write_all(&[0u8]).await.unwrap(); - s2.finish().await.unwrap(); + s2.finish().unwrap(); conn2.close( CONNECTION_CLOSE_CODE_DROPPED_ENTRY.into(), @@ -2308,7 +2315,7 @@ pub mod test { let mut send_stream = client_connection.open_uni().await.unwrap(); let data = format!("{i}").into_bytes(); send_stream.write_all(&data).await.unwrap(); - send_stream.finish().await.unwrap(); + send_stream.finish().unwrap(); } let elapsed_sending: f64 = start_time.elapsed().as_secs_f64(); info!("Elapsed sending: {elapsed_sending}"); diff --git a/streamer/src/nonblocking/testing_utilities.rs b/streamer/src/nonblocking/testing_utilities.rs index d0a1fa98d6d182..4a63458e7c6d74 100644 --- a/streamer/src/nonblocking/testing_utilities.rs +++ b/streamer/src/nonblocking/testing_utilities.rs @@ -10,7 +10,10 @@ use { tls_certificates::new_dummy_x509_certificate, }, crossbeam_channel::unbounded, - quinn::{ClientConfig, Connection, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}, + quinn::{ + crypto::rustls::QuicClientConfig, ClientConfig, Connection, EndpointConfig, IdleTimeout, + TokioRuntime, TransportConfig, + }, solana_perf::packet::PacketBatch, solana_sdk::{ net::DEFAULT_TPU_COALESCE, @@ -25,25 +28,57 @@ use { tokio::task::JoinHandle, }; -struct SkipServerVerification; +#[derive(Debug)] +pub struct SkipServerVerification(Arc); impl SkipServerVerification { - fn new() -> Arc { - Arc::new(Self) + pub fn new() -> Arc { + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } -impl rustls::client::ServerCertVerifier for SkipServerVerification { +impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + fn verify_server_cert( &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) } } @@ -51,15 +86,15 @@ pub fn get_client_config(keypair: &Keypair) -> ClientConfig { let (cert, key) = new_dummy_x509_certificate(keypair); let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() + .dangerous() .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key) - .expect("Provided key should be correctly set."); + .expect("Failed to use client certificate"); crypto.enable_early_data = true; crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(crypto)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(crypto).unwrap())); let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index a3c294236ae773..e9ca06a10bb133 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -5,8 +5,15 @@ use { }, crossbeam_channel::Sender, pem::Pem, - quinn::{Endpoint, IdleTimeout, ServerConfig}, - rustls::{server::ClientCertVerified, Certificate, DistinguishedName, KeyLogFile}, + quinn::{ + crypto::rustls::{NoInitialCipherSuite, QuicServerConfig}, + Endpoint, IdleTimeout, ServerConfig, + }, + rustls::{ + pki_types::{CertificateDer, UnixTime}, + server::danger::ClientCertVerified, + DistinguishedName, KeyLogFile, + }, solana_perf::packet::PacketBatch, solana_sdk::{ packet::PACKET_DATA_SIZE, @@ -20,7 +27,7 @@ use { Arc, Mutex, RwLock, }, thread, - time::{Duration, SystemTime}, + time::Duration, }, tokio::runtime::Runtime, }; @@ -31,11 +38,12 @@ pub const MAX_UNSTAKED_CONNECTIONS: usize = 500; // This will be adjusted and parameterized in follow-on PRs. pub const DEFAULT_QUIC_ENDPOINTS: usize = 1; -pub struct SkipClientVerification; +#[derive(Debug)] +pub struct SkipClientVerification(Arc); impl SkipClientVerification { pub fn new() -> Arc { - Arc::new(Self) + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } @@ -45,18 +53,58 @@ pub struct SpawnServerResult { pub key_updater: Arc, } -impl rustls::server::ClientCertVerifier for SkipClientVerification { - fn client_auth_root_subjects(&self) -> &[DistinguishedName] { +impl rustls::server::danger::ClientCertVerifier for SkipClientVerification { + fn verify_client_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _now: UnixTime, + ) -> Result { + Ok(rustls::server::danger::ClientCertVerified::assertion()) + } + + fn root_hint_subjects(&self) -> &[DistinguishedName] { &[] } - fn verify_client_cert( + fn verify_tls12_signature( &self, - _end_entity: &Certificate, - _intermediates: &[Certificate], - _now: SystemTime, - ) -> Result { - Ok(rustls::server::ClientCertVerified::assertion()) + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + + fn offer_client_auth(&self) -> bool { + true + } + + fn client_auth_mandatory(&self) -> bool { + self.offer_client_auth() } } @@ -64,25 +112,22 @@ impl rustls::server::ClientCertVerifier for SkipClientVerification { #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 pub(crate) fn configure_server( identity_keypair: &Keypair, - max_concurrent_connections: usize, ) -> Result<(ServerConfig, String), QuicServerError> { let (cert, priv_key) = new_dummy_x509_certificate(identity_keypair); let cert_chain_pem_parts = vec![Pem { tag: "CERTIFICATE".to_string(), - contents: cert.0.clone(), + contents: cert.as_ref().to_vec(), }]; let cert_chain_pem = pem::encode_many(&cert_chain_pem_parts); let mut server_tls_config = rustls::ServerConfig::builder() - .with_safe_defaults() .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], priv_key)?; server_tls_config.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; server_tls_config.key_log = Arc::new(KeyLogFile::new()); + let quic_server_config = QuicServerConfig::try_from(server_tls_config)?; - let mut server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); - server_config.concurrent_connections(max_concurrent_connections as u32); - server_config.use_retry(true); + let mut server_config = ServerConfig::with_crypto(Arc::new(quic_server_config)); let config = Arc::get_mut(&mut server_config.transport).unwrap(); // QUIC_MAX_CONCURRENT_STREAMS doubled, which was found to improve reliability @@ -122,16 +167,17 @@ pub enum QuicServerError { EndpointFailed(std::io::Error), #[error("TLS error: {0}")] TlsError(#[from] rustls::Error), + #[error("No initial cipher suite")] + NoInitialCipherSuite(#[from] NoInitialCipherSuite), } pub struct EndpointKeyUpdater { endpoints: Vec, - max_concurrent_connections: usize, } impl NotifyKeyUpdate for EndpointKeyUpdater { fn update_key(&self, key: &Keypair) -> Result<(), Box> { - let (config, _) = configure_server(key, self.max_concurrent_connections)?; + let (config, _) = configure_server(key)?; for endpoint in &self.endpoints { endpoint.set_server_config(Some(config.clone())); } @@ -632,7 +678,6 @@ pub fn spawn_server_multi( .unwrap(); let updater = EndpointKeyUpdater { endpoints: result.endpoints.clone(), - max_concurrent_connections: result.max_concurrent_connections, }; Ok(SpawnServerResult { endpoints: result.endpoints, diff --git a/streamer/src/tls_certificates.rs b/streamer/src/tls_certificates.rs index 866f6155abe3f6..fba1441de88a86 100644 --- a/streamer/src/tls_certificates.rs +++ b/streamer/src/tls_certificates.rs @@ -3,7 +3,12 @@ use { x509_parser::{prelude::*, public_key::PublicKey}, }; -pub fn new_dummy_x509_certificate(keypair: &Keypair) -> (rustls::Certificate, rustls::PrivateKey) { +pub fn new_dummy_x509_certificate( + keypair: &Keypair, +) -> ( + rustls::pki_types::CertificateDer<'static>, + rustls::pki_types::PrivateKeyDer<'static>, +) { // Unfortunately, rustls does not accept a "raw" Ed25519 key. // We have to convert it to DER and pass it to the library. @@ -91,12 +96,14 @@ pub fn new_dummy_x509_certificate(keypair: &Keypair) -> (rustls::Certificate, ru ]); ( - rustls::Certificate(cert_der), - rustls::PrivateKey(key_pkcs8_der), + rustls::pki_types::CertificateDer::from(cert_der), + rustls::pki_types::PrivateKeyDer::try_from(key_pkcs8_der).unwrap(), ) } -pub fn get_pubkey_from_tls_certificate(der_cert: &rustls::Certificate) -> Option { +pub fn get_pubkey_from_tls_certificate( + der_cert: &rustls::pki_types::CertificateDer, +) -> Option { let (_, cert) = X509Certificate::from_der(der_cert.as_ref()).ok()?; match cert.public_key().parsed().ok()? { PublicKey::Unknown(key) => Pubkey::try_from(key).ok(), diff --git a/turbine/src/quic_endpoint.rs b/turbine/src/quic_endpoint.rs index 40559b1ae8ef7c..47fb173838a5f1 100644 --- a/turbine/src/quic_endpoint.rs +++ b/turbine/src/quic_endpoint.rs @@ -4,11 +4,15 @@ use { futures::future::TryJoin, log::error, quinn::{ + crypto::rustls::{QuicClientConfig, QuicServerConfig}, ClientConfig, ConnectError, Connecting, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendDatagramError, ServerConfig, TokioRuntime, TransportConfig, VarInt, }, - rustls::{Certificate, KeyLogFile, PrivateKey}, + rustls::{ + pki_types::{CertificateDer, PrivateKeyDer}, + CertificateError, KeyLogFile, + }, solana_quic_client::nonblocking::quic_client::SkipServerVerification, solana_runtime::bank_forks::BankForks, solana_sdk::{pubkey::Pubkey, signature::Keypair}, @@ -102,7 +106,7 @@ pub fn new_quic_endpoint( Error, > { let (cert, key) = new_dummy_x509_certificate(keypair); - let server_config = new_server_config(cert.clone(), key.clone())?; + let server_config = new_server_config(cert.clone(), key.clone_key())?; let client_config = new_client_config(cert, key)?; let mut endpoint = { // Endpoint::new requires entering the runtime context, @@ -148,29 +152,36 @@ pub fn close_quic_endpoint(endpoint: &Endpoint) { ); } -fn new_server_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_server_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(SkipClientVerification {})) + .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], key)?; config.alpn_protocols = vec![ALPN_TURBINE_PROTOCOL_ID.to_vec()]; config.key_log = Arc::new(KeyLogFile::new()); - let mut config = ServerConfig::with_crypto(Arc::new(config)); + let quic_server_config = QuicServerConfig::try_from(config) + .map_err(|_err| rustls::Error::InvalidCertificate(CertificateError::BadSignature))?; + + let mut config = ServerConfig::with_crypto(Arc::new(quic_server_config)); config .transport_config(Arc::new(new_transport_config())) - .use_retry(true) .migration(false); Ok(config) } -fn new_client_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_client_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(Arc::new(SkipServerVerification {})) + .dangerous() + .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key)?; config.enable_early_data = true; config.alpn_protocols = vec![ALPN_TURBINE_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(config)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(config).unwrap())); config.transport_config(Arc::new(new_transport_config())); Ok(config) } @@ -202,17 +213,29 @@ async fn run_server( let stats = Arc::::default(); let report_metrics_task = tokio::task::spawn(report_metrics_task("turbine_quic_server", stats.clone())); - while let Some(connecting) = endpoint.accept().await { - tokio::task::spawn(handle_connecting_task( - endpoint.clone(), - connecting, - sender.clone(), - bank_forks.clone(), - prune_cache_pending.clone(), - router.clone(), - cache.clone(), - stats.clone(), - )); + while let Some(incoming) = endpoint.accept().await { + let remote_addr: SocketAddr = incoming.remote_address(); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::task::spawn(handle_connecting_task( + endpoint.clone(), + connecting, + sender.clone(), + bank_forks.clone(), + prune_cache_pending.clone(), + router.clone(), + cache.clone(), + stats.clone(), + )); + } + Err(error) => { + debug!( + "Error while accepting incoming connection: {error:?} from {}", + remote_addr + ); + } + } } report_metrics_task.abort(); } @@ -629,11 +652,15 @@ struct TurbineQuicStats { connection_error_timed_out: AtomicU64, connection_error_transport_error: AtomicU64, connection_error_version_mismatch: AtomicU64, + connection_error_connection_limit_exceeded: AtomicU64, invalid_identity: AtomicU64, router_try_send_error_full: AtomicU64, send_datagram_error_connection_lost: AtomicU64, send_datagram_error_too_large: AtomicU64, send_datagram_error_unsupported_by_peer: AtomicU64, + connect_error_cids_exhausted: AtomicU64, + connect_error_invalid_server_name: AtomicU64, + connection_error_cids_exhausted: AtomicU64, } async fn report_metrics_task(name: &'static str, stats: Arc) { @@ -649,12 +676,6 @@ fn record_error(err: &Error, stats: &TurbineQuicStats) { Error::ConnectError(ConnectError::EndpointStopping) => { add_metric!(stats.connect_error_other) } - Error::ConnectError(ConnectError::TooManyConnections) => { - add_metric!(stats.connect_error_too_many_connections) - } - Error::ConnectError(ConnectError::InvalidDnsName(_)) => { - add_metric!(stats.connect_error_other) - } Error::ConnectError(ConnectError::InvalidRemoteAddress(_)) => { add_metric!(stats.connect_error_invalid_remote_address) } @@ -696,6 +717,15 @@ fn record_error(err: &Error, stats: &TurbineQuicStats) { add_metric!(stats.send_datagram_error_connection_lost) } Error::TlsError(_) => (), + Error::ConnectError(ConnectError::CidsExhausted) => { + add_metric!(stats.connect_error_cids_exhausted) + } + Error::ConnectError(ConnectError::InvalidServerName(_)) => { + add_metric!(stats.connect_error_invalid_server_name) + } + Error::ConnectionError(ConnectionError::CidsExhausted) => { + add_metric!(stats.connection_error_cids_exhausted) + } } } @@ -757,6 +787,11 @@ fn report_metrics(name: &'static str, stats: &TurbineQuicStats) { reset_metric!(stats.connection_error_version_mismatch), i64 ), + ( + "connection_error_connection_limit_exceeded", + reset_metric!(stats.connection_error_connection_limit_exceeded), + i64 + ), ( "invalid_identity", reset_metric!(stats.invalid_identity),