diff --git a/.github/scripts/downstream-project-spl-common.sh b/.github/scripts/downstream-project-spl-common.sh index e70c10d070b7e9..f5e0a7e448f729 100644 --- a/.github/scripts/downstream-project-spl-common.sh +++ b/.github/scripts/downstream-project-spl-common.sh @@ -30,3 +30,7 @@ sed -i 's/solana-geyser-plugin-interface/agave-geyser-plugin-interface/g' ./Carg # should be removed when spl bump their curve25519-dalek sed -i "s/^curve25519-dalek =.*/curve25519-dalek = \"4.1.3\"/" token/confidential-transfer/proof-generation/Cargo.toml + +# fix curve25519-dalek + +sed -i '/\[patch.crates-io\]/a curve25519-dalek = { git = "https://github.com/anza-xyz/curve25519-dalek.git", rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" }' ./Cargo.toml diff --git a/Cargo.lock b/Cargo.lock index fb0f87b78b7640..0ce95129d77bea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -886,7 +886,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.77", ] @@ -1279,6 +1279,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1470,6 +1476,16 @@ dependencies = [ "unreachable", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1749,9 +1765,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "3.2.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder", "digest 0.9.0", @@ -2043,7 +2059,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ - "curve25519-dalek 3.2.1", + "curve25519-dalek 3.2.0", "ed25519", "rand 0.7.3", "serde", @@ -2877,7 +2893,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", ] @@ -3087,6 +3103,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine 4.6.7", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.24" @@ -3847,9 +3883,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "oorandom" @@ -4466,16 +4502,17 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quinn" -version = "0.10.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2 0.5.7", "thiserror", "tokio", "tracing", @@ -4483,16 +4520,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand 0.8.5", - "ring 0.16.20", - "rustc-hash", - "rustls", - "rustls-native-certs", + "ring 0.17.3", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "rustls-platform-verifier", "slab", "thiserror", "tinyvec", @@ -4501,15 +4538,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.4.0" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6df19e284d93757a9fb91d63672f7741b129246a669db09d1c0063071debc0c0" +checksum = "4fe68c2e9e1a1234e218683dbdf9f9dfcb094113c5ac2b938dfcb9bab4c4140b" dependencies = [ - "bytes", "libc", + "once_cell", "socket2 0.5.7", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -4800,7 +4837,7 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite", - "rustls", + "rustls 0.21.12", "rustls-pemfile 1.0.0", "serde", "serde_json", @@ -4916,6 +4953,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.3.3" @@ -4964,40 +5007,89 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring 0.17.3", + "rustls-pki-types", + "rustls-webpki 0.102.7", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.1" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca9ebdfa27d3fc180e42879037b5338ab1c040c06affd00d8338598e7800943" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile 0.2.1", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "0.2.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eebeaeb360c87bfb72e84abdb3447159c0eaececf1bef2aecd65a8be949d1c9" +checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" dependencies = [ "base64 0.13.1", ] [[package]] name = "rustls-pemfile" -version = "1.0.0" +version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7522c9de787ff061458fe9a829dc790a3f5b22dc571694fc5883f448b94d9a9" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" dependencies = [ - "base64 0.13.1", + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-platform-verifier" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "034f511ae2823d6a657925d5f7cf821d7458367ce0a54fbe743c471fdddd8111" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.102.7", + "security-framework", + "security-framework-sys", + "webpki-roots 0.26.5", + "winapi 0.3.9", ] +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -5008,6 +5100,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +dependencies = [ + "ring 0.17.3", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -5081,22 +5184,23 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.4.2" +version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "525bc1abfda2e1998d152c45cf13e696f76d0a4972310b22fac1658b05df7c87" +checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc" dependencies = [ "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", + "num-bigint 0.4.6", "security-framework-sys", ] [[package]] name = "security-framework-sys" -version = "2.4.2" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9dd14d83160b528b7bfd66439110573efcfbe281b17fc2ca9f39f550d619c7e" +checksum = "317936bbbd05227752583946b9e66d7ce3b489f84e11a94a510b4437fef407d7" dependencies = [ "core-foundation-sys", "libc", @@ -5442,9 +5546,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.5" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -6250,7 +6357,7 @@ dependencies = [ "rayon", "rolling-file", "rustc_version 0.4.1", - "rustls", + "rustls 0.23.12", "serde", "serde_bytes", "serde_derive", @@ -7298,7 +7405,7 @@ dependencies = [ "log", "quinn", "quinn-proto", - "rustls", + "rustls 0.23.12", "solana-connection-cache", "solana-logger", "solana-measure", @@ -7902,7 +8009,7 @@ dependencies = [ "quinn", "quinn-proto", "rand 0.8.5", - "rustls", + "rustls 0.23.12", "smallvec", "socket2 0.5.7", "solana-logger", @@ -8249,7 +8356,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rayon", - "rustls", + "rustls 0.23.12", "solana-entry", "solana-feature-set", "solana-gossip", @@ -8546,7 +8653,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c1941b5ef0c3ce8f2ac5dd984d0fb1a97423c4ff2a02eec81e3913f02e2ac2b" dependencies = [ "byteorder", - "combine", + "combine 3.8.1", "gdbstub", "hash32", "libc", @@ -9194,7 +9301,7 @@ dependencies = [ "once_cell", "pbkdf2 0.4.0", "rand 0.7.3", - "rustc-hash", + "rustc-hash 1.1.0", "sha2 0.9.9", "thiserror", "unicode-normalization", @@ -9282,7 +9389,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", "tokio", ] @@ -9321,7 +9428,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", "tungstenite", @@ -9568,7 +9675,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls", + "rustls 0.21.12", "sha1", "thiserror", "url 2.5.2", @@ -9878,7 +9985,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -9887,6 +9994,15 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "webpki-roots" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.2.2" @@ -10215,9 +10331,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 2b8cbd137f4e12..6b4eac9eff9fe8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -325,8 +325,8 @@ prost-types = "0.11.9" protobuf-src = "1.1.0" qstring = "0.7.2" qualifier_attr = { version = "0.2.2", default-features = false } -quinn = "0.10.2" -quinn-proto = "0.10.6" +quinn = "0.11.4" +quinn-proto = "0.11.7" quote = "1.0" rand = "0.8.5" rand_chacha = "0.3.1" @@ -338,7 +338,7 @@ reqwest-middleware = "0.2.5" rolling-file = "0.2.0" rpassword = "7.3" rustc_version = "0.4" -rustls = { version = "0.21.12", default-features = false, features = ["quic"] } +rustls = { version = "0.23.9", default-features = false } scopeguard = "1.2.0" semver = "1.0.23" seqlock = "0.2.0" @@ -522,7 +522,7 @@ winapi = "0.3.8" winreg = "0.50" x509-parser = "0.14.0" # See "zeroize versioning issues" below if you are updating this version. -zeroize = { version = "1.3", default-features = false } +zeroize = { version = "1.7", default-features = false } zstd = "0.13.2" [patch.crates-io] diff --git a/core/src/repair/quic_endpoint.rs b/core/src/repair/quic_endpoint.rs index abaf6d03483f10..5f4fea3f637e73 100644 --- a/core/src/repair/quic_endpoint.rs +++ b/core/src/repair/quic_endpoint.rs @@ -5,11 +5,15 @@ use { itertools::Itertools, log::error, quinn::{ - ClientConfig, ConnectError, Connecting, Connection, ConnectionError, Endpoint, - EndpointConfig, IdleTimeout, ReadError, ReadToEndError, RecvStream, SendStream, + crypto::rustls::{QuicClientConfig, QuicServerConfig}, + ClientConfig, ClosedStream, ConnectError, Connecting, Connection, ConnectionError, + Endpoint, EndpointConfig, IdleTimeout, ReadError, ReadToEndError, RecvStream, SendStream, ServerConfig, TokioRuntime, TransportConfig, VarInt, WriteError, }, - rustls::{Certificate, KeyLogFile, PrivateKey}, + rustls::{ + pki_types::{CertificateDer, PrivateKeyDer}, + CertificateError, KeyLogFile, + }, serde_bytes::ByteBuf, solana_quic_client::nonblocking::quic_client::SkipServerVerification, solana_runtime::bank_forks::BankForks, @@ -105,6 +109,8 @@ pub(crate) enum Error { TlsError(#[from] rustls::Error), #[error(transparent)] WriteError(#[from] WriteError), + #[error(transparent)] + ClosedStream(#[from] ClosedStream), } macro_rules! add_metric { @@ -122,7 +128,7 @@ pub(crate) fn new_quic_endpoint( bank_forks: Arc>, ) -> Result<(Endpoint, AsyncSender, AsyncTryJoinHandle), Error> { let (cert, key) = new_dummy_x509_certificate(keypair); - let server_config = new_server_config(cert.clone(), key.clone())?; + let server_config = new_server_config(cert.clone(), key.clone_key())?; let client_config = new_client_config(cert, key)?; let mut endpoint = { // Endpoint::new requires entering the runtime context, @@ -168,29 +174,36 @@ pub(crate) fn close_quic_endpoint(endpoint: &Endpoint) { ); } -fn new_server_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_server_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(SkipClientVerification {})) + .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], key)?; config.alpn_protocols = vec![ALPN_REPAIR_PROTOCOL_ID.to_vec()]; config.key_log = Arc::new(KeyLogFile::new()); - let mut config = ServerConfig::with_crypto(Arc::new(config)); + let quic_server_config = QuicServerConfig::try_from(config) + .map_err(|_err| rustls::Error::InvalidCertificate(CertificateError::BadSignature))?; + + let mut config = ServerConfig::with_crypto(Arc::new(quic_server_config)); config .transport_config(Arc::new(new_transport_config())) - .use_retry(true) .migration(false); Ok(config) } -fn new_client_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_client_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(Arc::new(SkipServerVerification {})) + .dangerous() + .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key)?; config.enable_early_data = true; config.alpn_protocols = vec![ALPN_REPAIR_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(config)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(config).unwrap())); config.transport_config(Arc::new(new_transport_config())); Ok(config) } @@ -219,17 +232,26 @@ async fn run_server( let stats = Arc::::default(); let report_metrics_task = tokio::task::spawn(report_metrics_task("repair_quic_server", stats.clone())); - while let Some(connecting) = endpoint.accept().await { - tokio::task::spawn(handle_connecting_task( - endpoint.clone(), - connecting, - remote_request_sender.clone(), - bank_forks.clone(), - prune_cache_pending.clone(), - router.clone(), - cache.clone(), - stats.clone(), - )); + while let Some(incoming) = endpoint.accept().await { + let remote_addr: SocketAddr = incoming.remote_address(); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::task::spawn(handle_connecting_task( + endpoint.clone(), + connecting, + remote_request_sender.clone(), + bank_forks.clone(), + prune_cache_pending.clone(), + router.clone(), + cache.clone(), + stats.clone(), + )); + } + Err(error) => { + debug!("Error while accepting incoming connection: {error:?} from {remote_addr}"); + } + } } report_metrics_task.abort(); } @@ -504,7 +526,7 @@ async fn handle_streams( send_stream.write_all(&size.to_le_bytes()).await?; send_stream.write_all(&chunk).await?; } - send_stream.finish().await.map_err(Error::from) + send_stream.finish().map_err(Error::from) } async fn send_requests_task( @@ -565,7 +587,7 @@ async fn send_request( const READ_TIMEOUT_DURATION: Duration = Duration::from_secs(10); let (mut send_stream, mut recv_stream) = connection.open_bi().await?; send_stream.write_all(&bytes).await?; - send_stream.finish().await?; + send_stream.finish()?; // Each response is at most PACKET_DATA_SIZE bytes and requires // an additional 8 bytes to encode its length. let size = PACKET_DATA_SIZE @@ -778,6 +800,7 @@ struct RepairQuicStats { connection_error_timed_out: AtomicU64, connection_error_transport_error: AtomicU64, connection_error_version_mismatch: AtomicU64, + connection_error_connection_limit_exceeded: AtomicU64, invalid_identity: AtomicU64, no_response_received: AtomicU64, read_to_end_error_connection_lost: AtomicU64, @@ -792,6 +815,12 @@ struct RepairQuicStats { write_error_stopped: AtomicU64, write_error_unknown_stream: AtomicU64, write_error_zero_rtt_rejected: AtomicU64, + connect_error_cids_exhausted: AtomicU64, + connect_error_invalid_server_name: AtomicU64, + connection_error_cids_exhausted: AtomicU64, + closed_streams: AtomicU64, + read_to_end_error_closed_stream: AtomicU64, + write_error_closed_stream: AtomicU64, } async fn report_metrics_task(name: &'static str, stats: Arc) { @@ -808,12 +837,6 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::ConnectError(ConnectError::EndpointStopping) => { add_metric!(stats.connect_error_other) } - Error::ConnectError(ConnectError::TooManyConnections) => { - add_metric!(stats.connect_error_too_many_connections) - } - Error::ConnectError(ConnectError::InvalidDnsName(_)) => { - add_metric!(stats.connect_error_other) - } Error::ConnectError(ConnectError::InvalidRemoteAddress(_)) => { add_metric!(stats.connect_error_invalid_remote_address) } @@ -851,9 +874,6 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::ReadToEndError(ReadToEndError::Read(ReadError::ConnectionLost(_))) => { add_metric!(stats.read_to_end_error_connection_lost) } - Error::ReadToEndError(ReadToEndError::Read(ReadError::UnknownStream)) => { - add_metric!(stats.read_to_end_error_unknown_stream) - } Error::ReadToEndError(ReadToEndError::Read(ReadError::IllegalOrderedRead)) => { add_metric!(stats.read_to_end_error_illegal_ordered_read) } @@ -869,12 +889,27 @@ fn record_error(err: &Error, stats: &RepairQuicStats) { Error::WriteError(WriteError::ConnectionLost(_)) => { add_metric!(stats.write_error_connection_lost) } - Error::WriteError(WriteError::UnknownStream) => { - add_metric!(stats.write_error_unknown_stream) - } Error::WriteError(WriteError::ZeroRttRejected) => { add_metric!(stats.write_error_zero_rtt_rejected) } + Error::ConnectError(ConnectError::CidsExhausted) => { + add_metric!(stats.connect_error_cids_exhausted) + } + Error::ConnectError(ConnectError::InvalidServerName(_)) => { + add_metric!(stats.connect_error_invalid_server_name) + } + Error::ConnectionError(ConnectionError::CidsExhausted) => { + add_metric!(stats.connection_error_cids_exhausted) + } + Error::ClosedStream(_) => { + add_metric!(stats.closed_streams) + } + Error::ReadToEndError(ReadToEndError::Read(ReadError::ClosedStream)) => { + add_metric!(stats.read_to_end_error_closed_stream) + } + Error::WriteError(WriteError::ClosedStream) => { + add_metric!(stats.write_error_closed_stream) + } } } @@ -936,6 +971,11 @@ fn report_metrics(name: &'static str, stats: &RepairQuicStats) { reset_metric!(stats.connection_error_version_mismatch), i64 ), + ( + "connection_error_connection_limit_exceeded", + reset_metric!(stats.connection_error_connection_limit_exceeded), + i64 + ), ( "invalid_identity", reset_metric!(stats.invalid_identity), diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index cbe090c6449fe3..c275a7fe049f48 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -662,7 +662,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash", + "rustc-hash 1.1.0", "shlex", "syn 2.0.58", ] @@ -927,6 +927,12 @@ dependencies = [ "libc", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -1027,6 +1033,16 @@ dependencies = [ "unreachable", ] +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -1207,9 +1223,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "3.2.1" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f9d052967f590a76e62eb387bd0bbb1b000182c3cefe5364db6b7211651bc0" +checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61" dependencies = [ "byteorder 1.5.0", "digest 0.9.0", @@ -1484,7 +1500,7 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c762bae6dcaf24c4c84667b8579785430908723d5c889f469d76a41d59cc7a9d" dependencies = [ - "curve25519-dalek 3.2.1", + "curve25519-dalek 3.2.0", "ed25519", "rand 0.7.3", "serde", @@ -2209,7 +2225,7 @@ dependencies = [ "futures-util", "http", "hyper", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", ] @@ -2408,6 +2424,26 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" +[[package]] +name = "jni" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6df18c2e3db7e453d3c6ac5b3e9d5182664d28788126d39b91f2d1e22b017ec" +dependencies = [ + "cesu8", + "combine 4.6.7", + "jni-sys", + "log", + "thiserror", + "walkdir", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + [[package]] name = "jobserver" version = "0.1.21" @@ -3208,9 +3244,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "opaque-debug" @@ -3739,16 +3775,17 @@ dependencies = [ [[package]] name = "quinn" -version = "0.10.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cc2c5017e4b43d5995dcea317bc46c1e09404c0a9664d2908f7f02dfe943d75" +checksum = "8c7c5fdde3cdae7203427dc4f0a68fe0ed09833edc525a03456b153b79828684" dependencies = [ "bytes", "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", - "rustls", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "socket2 0.5.7", "thiserror", "tokio", "tracing", @@ -3756,16 +3793,16 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.10.6" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "141bf7dfde2fbc246bfd3fe12f2455aa24b0fbd9af535d8c86c7bd1381ff2b1a" +checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", "rand 0.8.5", - "ring 0.16.20", - "rustc-hash", - "rustls", - "rustls-native-certs", + "ring 0.17.3", + "rustc-hash 2.0.0", + "rustls 0.23.12", + "rustls-platform-verifier", "slab", "thiserror", "tinyvec", @@ -3774,15 +3811,15 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.4.1" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "055b4e778e8feb9f93c4e439f71dc2156ef13360b432b799e179a8c4cdf0b1d7" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" dependencies = [ - "bytes", "libc", + "once_cell", "socket2 0.5.7", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -4008,8 +4045,8 @@ dependencies = [ "once_cell", "percent-encoding 2.3.1", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.21.12", + "rustls-pemfile 1.0.0", "serde", "serde_json", "serde_urlencoded", @@ -4124,6 +4161,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc-hash" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152" + [[package]] name = "rustc_version" version = "0.4.0" @@ -4163,18 +4206,33 @@ checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" dependencies = [ "log", "ring 0.17.3", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "once_cell", + "ring 0.17.3", + "rustls-pki-types", + "rustls-webpki 0.102.7", + "subtle", + "zeroize", +] + [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "e5bfb394eeed242e909609f56089eecfe5fda225042e8b171791b9c95f5931e5" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pemfile 2.1.3", + "rustls-pki-types", "schannel", "security-framework", ] @@ -4188,6 +4246,49 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64 0.22.1", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-platform-verifier" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbb878bdfdf63a336a5e63561b1835e7a8c91524f51621db870169eac84b490" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls 0.23.12", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki 0.102.7", + "security-framework", + "security-framework-sys", + "webpki-roots 0.26.5", + "winapi 0.3.9", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -4198,6 +4299,17 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84678086bd54edf2b415183ed7a94d0efb049f1b646a33e22a36f3794be6ae56" +dependencies = [ + "ring 0.17.3", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" version = "1.0.17" @@ -4260,6 +4372,7 @@ dependencies = [ "core-foundation", "core-foundation-sys", "libc", + "num-bigint 0.4.6", "security-framework-sys", ] @@ -4504,9 +4617,12 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.2" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] [[package]] name = "smallvec" @@ -4959,7 +5075,7 @@ dependencies = [ "rand_chacha 0.3.1", "rayon", "rolling-file", - "rustls", + "rustls 0.23.12", "serde", "serde_bytes", "serde_derive", @@ -5661,7 +5777,7 @@ dependencies = [ "log", "quinn", "quinn-proto", - "rustls", + "rustls 0.23.12", "solana-connection-cache", "solana-measure", "solana-metrics", @@ -6603,7 +6719,7 @@ dependencies = [ "quinn", "quinn-proto", "rand 0.8.5", - "rustls", + "rustls 0.23.12", "smallvec", "socket2 0.5.7", "solana-measure", @@ -6819,7 +6935,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rayon", - "rustls", + "rustls 0.23.12", "solana-entry", "solana-feature-set", "solana-gossip", @@ -7047,7 +7163,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c1941b5ef0c3ce8f2ac5dd984d0fb1a97423c4ff2a02eec81e3913f02e2ac2b" dependencies = [ "byteorder 1.5.0", - "combine", + "combine 3.8.1", "hash32", "libc", "log", @@ -7630,7 +7746,7 @@ dependencies = [ "once_cell", "pbkdf2 0.4.0", "rand 0.7.3", - "rustc-hash", + "rustc-hash 1.1.0", "sha2 0.9.9", "thiserror", "unicode-normalization", @@ -7710,7 +7826,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.12", "tokio", ] @@ -7749,7 +7865,7 @@ checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" dependencies = [ "futures-util", "log", - "rustls", + "rustls 0.21.12", "tokio", "tokio-rustls", "tungstenite", @@ -7833,7 +7949,7 @@ dependencies = [ "percent-encoding 2.3.1", "pin-project", "prost", - "rustls-pemfile", + "rustls-pemfile 1.0.0", "tokio", "tokio-rustls", "tokio-stream", @@ -7971,7 +8087,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "rustls", + "rustls 0.21.12", "sha1", "thiserror", "url 2.5.2", @@ -8260,7 +8376,7 @@ version = "0.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b291546d5d9d1eab74f069c77749f2cb8504a12caa20f0f2de93ddbf6f411888" dependencies = [ - "rustls-webpki", + "rustls-webpki 0.101.7", ] [[package]] @@ -8269,6 +8385,15 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14247bb57be4f377dfb94c72830b8ce8fc6beac03cf4bf7b9732eadd414123fc" +[[package]] +name = "webpki-roots" +version = "0.26.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd24728e5af82c6c4ec1b66ac4844bdf8156257fccda846ec58b42cd0cdbe6a" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" version = "4.2.5" @@ -8525,9 +8650,9 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.3.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4756f7db3f7b5574938c3eb1c117038b8e07f95ee6718c0efad4ac21508f1efd" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" dependencies = [ "zeroize_derive", ] diff --git a/quic-client/Cargo.toml b/quic-client/Cargo.toml index 05800896c30f80..428b03cb0a6892 100644 --- a/quic-client/Cargo.toml +++ b/quic-client/Cargo.toml @@ -18,7 +18,7 @@ lazy_static = { workspace = true } log = { workspace = true } quinn = { workspace = true } quinn-proto = { workspace = true } -rustls = { workspace = true, features = ["dangerous_configuration"] } +rustls = { workspace = true } solana-connection-cache = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/quic-client/src/nonblocking/quic_client.rs b/quic-client/src/nonblocking/quic_client.rs index b40e89ec216c9d..c6a66fe56bcd10 100644 --- a/quic-client/src/nonblocking/quic_client.rs +++ b/quic-client/src/nonblocking/quic_client.rs @@ -8,8 +8,9 @@ use { itertools::Itertools, log::*, quinn::{ - ClientConfig, ConnectError, Connection, ConnectionError, Endpoint, EndpointConfig, - IdleTimeout, TokioRuntime, TransportConfig, WriteError, + crypto::rustls::QuicClientConfig, ClientConfig, ClosedStream, ConnectError, Connection, + ConnectionError, Endpoint, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig, + WriteError, }, solana_connection_cache::{ client_connection::ClientStats, connection_cache_stats::ConnectionCacheStats, @@ -38,31 +39,63 @@ use { tokio::{sync::OnceCell, time::timeout}, }; -pub struct SkipServerVerification; +#[derive(Debug)] +pub struct SkipServerVerification(Arc); impl SkipServerVerification { pub fn new() -> Arc { - Arc::new(Self) + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } -impl rustls::client::ServerCertVerifier for SkipServerVerification { +impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + fn verify_server_cert( &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) } } pub struct QuicClientCertificate { - pub certificate: rustls::Certificate, - pub key: rustls::PrivateKey, + pub certificate: rustls::pki_types::CertificateDer<'static>, + pub key: rustls::pki_types::PrivateKeyDer<'static>, } /// A lazy-initialized Quic Endpoint @@ -80,6 +113,8 @@ pub enum QuicError { ConnectionError(#[from] ConnectionError), #[error(transparent)] ConnectError(#[from] ConnectError), + #[error(transparent)] + ClosedStream(#[from] ClosedStream), } impl From for ClientErrorKind { @@ -115,17 +150,17 @@ impl QuicLazyInitializedEndpoint { }; let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() + .dangerous() .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert( vec![self.client_certificate.certificate.clone()], - self.client_certificate.key.clone(), + self.client_certificate.key.clone_key(), ) .expect("Failed to set QUIC client certificates"); crypto.enable_early_data = true; crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(crypto)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(crypto).unwrap())); let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); diff --git a/scripts/build-downstream-anchor-projects.sh b/scripts/build-downstream-anchor-projects.sh index e282a446c7c79a..fe2f5e813ee613 100755 --- a/scripts/build-downstream-anchor-projects.sh +++ b/scripts/build-downstream-anchor-projects.sh @@ -69,6 +69,7 @@ anchor() { patch_crates_io_solana Cargo.toml "$solana_dir" patch_spl_crates . Cargo.toml "$spl_dir" + sed -i '/\[patch.crates-io\]/a curve25519-dalek = { git = "https://github.com/anza-xyz/curve25519-dalek.git", rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464" }' ./Cargo.toml $cargo test # serum_dex and mpl-token-metadata are using caret versions of solana and SPL dependencies # rather pull and patch those as well, ignore for now diff --git a/streamer/Cargo.toml b/streamer/Cargo.toml index 89ce80c910c80f..873cb4459e3327 100644 --- a/streamer/Cargo.toml +++ b/streamer/Cargo.toml @@ -28,7 +28,7 @@ percentage = { workspace = true } quinn = { workspace = true } quinn-proto = { workspace = true } rand = { workspace = true } -rustls = { workspace = true, features = ["dangerous_configuration"] } +rustls = { workspace = true } smallvec = { workspace = true } socket2 = { workspace = true } solana-measure = { workspace = true } diff --git a/streamer/src/nonblocking/quic.rs b/streamer/src/nonblocking/quic.rs index f1b0a5a7efd5ed..760902a6aab06b 100644 --- a/streamer/src/nonblocking/quic.rs +++ b/streamer/src/nonblocking/quic.rs @@ -198,7 +198,7 @@ pub fn spawn_server_multi( let concurrent_connections = (max_staked_connections + max_unstaked_connections) / sockets.len(); let max_concurrent_connections = concurrent_connections + concurrent_connections / 4; - let (config, _cert) = configure_server(keypair, max_concurrent_connections)?; + let (config, _) = configure_server(keypair)?; let endpoints = sockets .into_iter() @@ -239,7 +239,7 @@ pub fn spawn_server_multi( #[allow(clippy::too_many_arguments)] async fn run_server( name: &'static str, - incoming: Vec, + endpoints: Vec, packet_sender: Sender, exit: Arc, max_connections_per_peer: usize, @@ -268,7 +268,7 @@ async fn run_server( )); stats .quic_endpoints_count - .store(incoming.len(), Ordering::Relaxed); + .store(endpoints.len(), Ordering::Relaxed); let staked_connection_table: Arc> = Arc::new(Mutex::new(ConnectionTable::new())); let (sender, receiver) = async_unbounded(); @@ -280,7 +280,7 @@ async fn run_server( coalesce, )); - let mut accepts = incoming + let mut accepts = endpoints .iter() .enumerate() .map(|(i, incoming)| { @@ -296,7 +296,7 @@ async fn run_server( if let Some((connecting, i)) = ready { accepts.push( Box::pin(EndpointAccept { - accept: incoming[i].accept(), + accept: endpoints[i].accept(), endpoint: i, } )); @@ -316,11 +316,11 @@ async fn run_server( last_datapoint = Instant::now(); } - if let Ok(Some(connection)) = timeout_connection { + if let Ok(Some(incoming)) = timeout_connection { stats .total_incoming_connection_attempts .fetch_add(1, Ordering::Relaxed); - let remote_address = connection.remote_address(); + let remote_address = incoming.remote_address(); // first check overall connection rate limit: if !overall_connection_rate_limiter.is_allowed() { @@ -331,6 +331,7 @@ async fn run_server( stats .connection_rate_limited_across_all .fetch_add(1, Ordering::Relaxed); + incoming.ignore(); continue; } @@ -349,26 +350,35 @@ async fn run_server( stats .connection_rate_limited_per_ipaddr .fetch_add(1, Ordering::Relaxed); + incoming.ignore(); continue; } stats .outstanding_incoming_connection_attempts .fetch_add(1, Ordering::Relaxed); - tokio::spawn(setup_connection( - connection, - unstaked_connection_table.clone(), - staked_connection_table.clone(), - sender.clone(), - max_connections_per_peer, - staked_nodes.clone(), - max_staked_connections, - max_unstaked_connections, - max_streams_per_ms, - stats.clone(), - wait_for_chunk_timeout, - stream_load_ema.clone(), - )); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::spawn(setup_connection( + connecting, + unstaked_connection_table.clone(), + staked_connection_table.clone(), + sender.clone(), + max_connections_per_peer, + staked_nodes.clone(), + max_staked_connections, + max_unstaked_connections, + max_streams_per_ms, + stats.clone(), + wait_for_chunk_timeout, + stream_load_ema.clone(), + )); + } + Err(err) => { + debug!("Incoming::accept(): error {:?}", err); + } + } } else { debug!("accept(): Timed out waiting for connection"); } @@ -394,7 +404,7 @@ pub fn get_remote_pubkey(connection: &Connection) -> Option { // Use the client cert only if it is self signed and the chain length is 1. connection .peer_identity()? - .downcast::>() + .downcast::>() .ok() .filter(|certs| certs.len() == 1)? .first() @@ -1263,7 +1273,7 @@ impl Drop for ConnectionEntry { } } -#[derive(Copy, Clone, Eq, Hash, PartialEq)] +#[derive(Copy, Clone, Debug, Eq, Hash, PartialEq)] enum ConnectionTableKey { IP(IpAddr), Pubkey(Pubkey), @@ -1422,7 +1432,7 @@ struct EndpointAccept<'a> { } impl<'a> Future for EndpointAccept<'a> { - type Output = (Option, usize); + type Output = (Option, usize); fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context) -> Poll { let i = self.endpoint; @@ -1463,7 +1473,7 @@ pub mod test { for i in 0..total { let mut s1 = conn1.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); info!("done {}", i); sleep(Duration::from_millis(1000)).await; } @@ -1488,7 +1498,7 @@ pub mod test { let s2 = conn2.open_uni().await; if let Ok(mut s2) = s2 { s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); // Send enough data to create more than 1 chunks. // The first will try to open the connection (which should fail). // The following chunks will enable the detection of connection failure. @@ -1496,9 +1506,6 @@ pub mod test { s2.write_all(&data) .await .expect_err("shouldn't be able to open 2 connections"); - s2.finish() - .await - .expect_err("shouldn't be able to open 2 connections"); } else { // It has been noticed if there is already connection open against the server, this open_uni can fail // with ApplicationClosed(ApplicationClose) error due to CONNECTION_CLOSE_CODE_TOO_MANY before writing to @@ -1521,9 +1528,9 @@ pub mod test { let mut s1 = c1.open_uni().await.unwrap(); let mut s2 = c2.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); s2.write_all(&[0u8]).await.unwrap(); - s2.finish().await.unwrap(); + s2.finish().unwrap(); num_expected_packets += 2; sleep(Duration::from_millis(200)).await; } @@ -1563,7 +1570,7 @@ pub mod test { for _ in 0..num_bytes { s1.write_all(&[0u8]).await.unwrap(); } - s1.finish().await.unwrap(); + s1.finish().unwrap(); let mut all_packets = vec![]; let now = Instant::now(); @@ -1598,7 +1605,8 @@ pub mod test { // Ignoring any errors here. s1.finish() will test the error condition s1.write_all(&[0u8]).await.unwrap_or_default(); } - s1.finish().await.unwrap_err(); + s1.finish().unwrap_or_default(); + s1.stopped().await.unwrap_err(); } } @@ -1712,7 +1720,6 @@ pub mod test { // Test that more writes to the stream will fail (i.e. the stream is no longer writable // after the timeouts) assert!(s1.write_all(&[0u8]).await.is_err()); - assert!(s1.finish().await.is_err()); exit.store(true, Ordering::Relaxed); join_handle.await.unwrap(); @@ -1775,7 +1782,7 @@ pub mod test { let mut s1 = conn1.open_uni().await.unwrap(); s1.write_all(&[0u8]).await.unwrap(); - s1.finish().await.unwrap(); + s1.finish().unwrap(); let mut s2 = conn2.open_uni().await.unwrap(); conn1.close( @@ -1789,7 +1796,7 @@ pub mod test { assert_eq!(stats.connection_removed.load(Ordering::Relaxed), 1); s2.write_all(&[0u8]).await.unwrap(); - s2.finish().await.unwrap(); + s2.finish().unwrap(); conn2.close( CONNECTION_CLOSE_CODE_DROPPED_ENTRY.into(), @@ -2308,7 +2315,7 @@ pub mod test { let mut send_stream = client_connection.open_uni().await.unwrap(); let data = format!("{i}").into_bytes(); send_stream.write_all(&data).await.unwrap(); - send_stream.finish().await.unwrap(); + send_stream.finish().unwrap(); } let elapsed_sending: f64 = start_time.elapsed().as_secs_f64(); info!("Elapsed sending: {elapsed_sending}"); diff --git a/streamer/src/nonblocking/testing_utilities.rs b/streamer/src/nonblocking/testing_utilities.rs index d0a1fa98d6d182..4a63458e7c6d74 100644 --- a/streamer/src/nonblocking/testing_utilities.rs +++ b/streamer/src/nonblocking/testing_utilities.rs @@ -10,7 +10,10 @@ use { tls_certificates::new_dummy_x509_certificate, }, crossbeam_channel::unbounded, - quinn::{ClientConfig, Connection, EndpointConfig, IdleTimeout, TokioRuntime, TransportConfig}, + quinn::{ + crypto::rustls::QuicClientConfig, ClientConfig, Connection, EndpointConfig, IdleTimeout, + TokioRuntime, TransportConfig, + }, solana_perf::packet::PacketBatch, solana_sdk::{ net::DEFAULT_TPU_COALESCE, @@ -25,25 +28,57 @@ use { tokio::task::JoinHandle, }; -struct SkipServerVerification; +#[derive(Debug)] +pub struct SkipServerVerification(Arc); impl SkipServerVerification { - fn new() -> Arc { - Arc::new(Self) + pub fn new() -> Arc { + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } -impl rustls::client::ServerCertVerifier for SkipServerVerification { +impl rustls::client::danger::ServerCertVerifier for SkipServerVerification { + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + fn verify_server_cert( &self, - _end_entity: &rustls::Certificate, - _intermediates: &[rustls::Certificate], - _server_name: &rustls::ServerName, - _scts: &mut dyn Iterator, + _end_entity: &rustls::pki_types::CertificateDer<'_>, + _intermediates: &[rustls::pki_types::CertificateDer<'_>], + _server_name: &rustls::pki_types::ServerName<'_>, _ocsp_response: &[u8], - _now: std::time::SystemTime, - ) -> Result { - Ok(rustls::client::ServerCertVerified::assertion()) + _now: rustls::pki_types::UnixTime, + ) -> Result { + Ok(rustls::client::danger::ServerCertVerified::assertion()) } } @@ -51,15 +86,15 @@ pub fn get_client_config(keypair: &Keypair) -> ClientConfig { let (cert, key) = new_dummy_x509_certificate(keypair); let mut crypto = rustls::ClientConfig::builder() - .with_safe_defaults() + .dangerous() .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key) - .expect("Provided key should be correctly set."); + .expect("Failed to use client certificate"); crypto.enable_early_data = true; crypto.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(crypto)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(crypto).unwrap())); let mut transport_config = TransportConfig::default(); let timeout = IdleTimeout::try_from(QUIC_MAX_TIMEOUT).unwrap(); diff --git a/streamer/src/quic.rs b/streamer/src/quic.rs index a3c294236ae773..e9ca06a10bb133 100644 --- a/streamer/src/quic.rs +++ b/streamer/src/quic.rs @@ -5,8 +5,15 @@ use { }, crossbeam_channel::Sender, pem::Pem, - quinn::{Endpoint, IdleTimeout, ServerConfig}, - rustls::{server::ClientCertVerified, Certificate, DistinguishedName, KeyLogFile}, + quinn::{ + crypto::rustls::{NoInitialCipherSuite, QuicServerConfig}, + Endpoint, IdleTimeout, ServerConfig, + }, + rustls::{ + pki_types::{CertificateDer, UnixTime}, + server::danger::ClientCertVerified, + DistinguishedName, KeyLogFile, + }, solana_perf::packet::PacketBatch, solana_sdk::{ packet::PACKET_DATA_SIZE, @@ -20,7 +27,7 @@ use { Arc, Mutex, RwLock, }, thread, - time::{Duration, SystemTime}, + time::Duration, }, tokio::runtime::Runtime, }; @@ -31,11 +38,12 @@ pub const MAX_UNSTAKED_CONNECTIONS: usize = 500; // This will be adjusted and parameterized in follow-on PRs. pub const DEFAULT_QUIC_ENDPOINTS: usize = 1; -pub struct SkipClientVerification; +#[derive(Debug)] +pub struct SkipClientVerification(Arc); impl SkipClientVerification { pub fn new() -> Arc { - Arc::new(Self) + Arc::new(Self(Arc::new(rustls::crypto::ring::default_provider()))) } } @@ -45,18 +53,58 @@ pub struct SpawnServerResult { pub key_updater: Arc, } -impl rustls::server::ClientCertVerifier for SkipClientVerification { - fn client_auth_root_subjects(&self) -> &[DistinguishedName] { +impl rustls::server::danger::ClientCertVerifier for SkipClientVerification { + fn verify_client_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _now: UnixTime, + ) -> Result { + Ok(rustls::server::danger::ClientCertVerified::assertion()) + } + + fn root_hint_subjects(&self) -> &[DistinguishedName] { &[] } - fn verify_client_cert( + fn verify_tls12_signature( &self, - _end_entity: &Certificate, - _intermediates: &[Certificate], - _now: SystemTime, - ) -> Result { - Ok(rustls::server::ClientCertVerified::assertion()) + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &rustls::pki_types::CertificateDer<'_>, + dss: &rustls::DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature( + message, + cert, + dss, + &self.0.signature_verification_algorithms, + ) + } + + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } + + fn offer_client_auth(&self) -> bool { + true + } + + fn client_auth_mandatory(&self) -> bool { + self.offer_client_auth() } } @@ -64,25 +112,22 @@ impl rustls::server::ClientCertVerifier for SkipClientVerification { #[allow(clippy::field_reassign_with_default)] // https://github.com/rust-lang/rust-clippy/issues/6527 pub(crate) fn configure_server( identity_keypair: &Keypair, - max_concurrent_connections: usize, ) -> Result<(ServerConfig, String), QuicServerError> { let (cert, priv_key) = new_dummy_x509_certificate(identity_keypair); let cert_chain_pem_parts = vec![Pem { tag: "CERTIFICATE".to_string(), - contents: cert.0.clone(), + contents: cert.as_ref().to_vec(), }]; let cert_chain_pem = pem::encode_many(&cert_chain_pem_parts); let mut server_tls_config = rustls::ServerConfig::builder() - .with_safe_defaults() .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], priv_key)?; server_tls_config.alpn_protocols = vec![ALPN_TPU_PROTOCOL_ID.to_vec()]; server_tls_config.key_log = Arc::new(KeyLogFile::new()); + let quic_server_config = QuicServerConfig::try_from(server_tls_config)?; - let mut server_config = ServerConfig::with_crypto(Arc::new(server_tls_config)); - server_config.concurrent_connections(max_concurrent_connections as u32); - server_config.use_retry(true); + let mut server_config = ServerConfig::with_crypto(Arc::new(quic_server_config)); let config = Arc::get_mut(&mut server_config.transport).unwrap(); // QUIC_MAX_CONCURRENT_STREAMS doubled, which was found to improve reliability @@ -122,16 +167,17 @@ pub enum QuicServerError { EndpointFailed(std::io::Error), #[error("TLS error: {0}")] TlsError(#[from] rustls::Error), + #[error("No initial cipher suite")] + NoInitialCipherSuite(#[from] NoInitialCipherSuite), } pub struct EndpointKeyUpdater { endpoints: Vec, - max_concurrent_connections: usize, } impl NotifyKeyUpdate for EndpointKeyUpdater { fn update_key(&self, key: &Keypair) -> Result<(), Box> { - let (config, _) = configure_server(key, self.max_concurrent_connections)?; + let (config, _) = configure_server(key)?; for endpoint in &self.endpoints { endpoint.set_server_config(Some(config.clone())); } @@ -632,7 +678,6 @@ pub fn spawn_server_multi( .unwrap(); let updater = EndpointKeyUpdater { endpoints: result.endpoints.clone(), - max_concurrent_connections: result.max_concurrent_connections, }; Ok(SpawnServerResult { endpoints: result.endpoints, diff --git a/streamer/src/tls_certificates.rs b/streamer/src/tls_certificates.rs index 866f6155abe3f6..fba1441de88a86 100644 --- a/streamer/src/tls_certificates.rs +++ b/streamer/src/tls_certificates.rs @@ -3,7 +3,12 @@ use { x509_parser::{prelude::*, public_key::PublicKey}, }; -pub fn new_dummy_x509_certificate(keypair: &Keypair) -> (rustls::Certificate, rustls::PrivateKey) { +pub fn new_dummy_x509_certificate( + keypair: &Keypair, +) -> ( + rustls::pki_types::CertificateDer<'static>, + rustls::pki_types::PrivateKeyDer<'static>, +) { // Unfortunately, rustls does not accept a "raw" Ed25519 key. // We have to convert it to DER and pass it to the library. @@ -91,12 +96,14 @@ pub fn new_dummy_x509_certificate(keypair: &Keypair) -> (rustls::Certificate, ru ]); ( - rustls::Certificate(cert_der), - rustls::PrivateKey(key_pkcs8_der), + rustls::pki_types::CertificateDer::from(cert_der), + rustls::pki_types::PrivateKeyDer::try_from(key_pkcs8_der).unwrap(), ) } -pub fn get_pubkey_from_tls_certificate(der_cert: &rustls::Certificate) -> Option { +pub fn get_pubkey_from_tls_certificate( + der_cert: &rustls::pki_types::CertificateDer, +) -> Option { let (_, cert) = X509Certificate::from_der(der_cert.as_ref()).ok()?; match cert.public_key().parsed().ok()? { PublicKey::Unknown(key) => Pubkey::try_from(key).ok(), diff --git a/turbine/src/quic_endpoint.rs b/turbine/src/quic_endpoint.rs index 40559b1ae8ef7c..47fb173838a5f1 100644 --- a/turbine/src/quic_endpoint.rs +++ b/turbine/src/quic_endpoint.rs @@ -4,11 +4,15 @@ use { futures::future::TryJoin, log::error, quinn::{ + crypto::rustls::{QuicClientConfig, QuicServerConfig}, ClientConfig, ConnectError, Connecting, Connection, ConnectionError, Endpoint, EndpointConfig, IdleTimeout, SendDatagramError, ServerConfig, TokioRuntime, TransportConfig, VarInt, }, - rustls::{Certificate, KeyLogFile, PrivateKey}, + rustls::{ + pki_types::{CertificateDer, PrivateKeyDer}, + CertificateError, KeyLogFile, + }, solana_quic_client::nonblocking::quic_client::SkipServerVerification, solana_runtime::bank_forks::BankForks, solana_sdk::{pubkey::Pubkey, signature::Keypair}, @@ -102,7 +106,7 @@ pub fn new_quic_endpoint( Error, > { let (cert, key) = new_dummy_x509_certificate(keypair); - let server_config = new_server_config(cert.clone(), key.clone())?; + let server_config = new_server_config(cert.clone(), key.clone_key())?; let client_config = new_client_config(cert, key)?; let mut endpoint = { // Endpoint::new requires entering the runtime context, @@ -148,29 +152,36 @@ pub fn close_quic_endpoint(endpoint: &Endpoint) { ); } -fn new_server_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_server_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ServerConfig::builder() - .with_safe_defaults() - .with_client_cert_verifier(Arc::new(SkipClientVerification {})) + .with_client_cert_verifier(SkipClientVerification::new()) .with_single_cert(vec![cert], key)?; config.alpn_protocols = vec![ALPN_TURBINE_PROTOCOL_ID.to_vec()]; config.key_log = Arc::new(KeyLogFile::new()); - let mut config = ServerConfig::with_crypto(Arc::new(config)); + let quic_server_config = QuicServerConfig::try_from(config) + .map_err(|_err| rustls::Error::InvalidCertificate(CertificateError::BadSignature))?; + + let mut config = ServerConfig::with_crypto(Arc::new(quic_server_config)); config .transport_config(Arc::new(new_transport_config())) - .use_retry(true) .migration(false); Ok(config) } -fn new_client_config(cert: Certificate, key: PrivateKey) -> Result { +fn new_client_config( + cert: CertificateDer<'static>, + key: PrivateKeyDer<'static>, +) -> Result { let mut config = rustls::ClientConfig::builder() - .with_safe_defaults() - .with_custom_certificate_verifier(Arc::new(SkipServerVerification {})) + .dangerous() + .with_custom_certificate_verifier(SkipServerVerification::new()) .with_client_auth_cert(vec![cert], key)?; config.enable_early_data = true; config.alpn_protocols = vec![ALPN_TURBINE_PROTOCOL_ID.to_vec()]; - let mut config = ClientConfig::new(Arc::new(config)); + let mut config = ClientConfig::new(Arc::new(QuicClientConfig::try_from(config).unwrap())); config.transport_config(Arc::new(new_transport_config())); Ok(config) } @@ -202,17 +213,29 @@ async fn run_server( let stats = Arc::::default(); let report_metrics_task = tokio::task::spawn(report_metrics_task("turbine_quic_server", stats.clone())); - while let Some(connecting) = endpoint.accept().await { - tokio::task::spawn(handle_connecting_task( - endpoint.clone(), - connecting, - sender.clone(), - bank_forks.clone(), - prune_cache_pending.clone(), - router.clone(), - cache.clone(), - stats.clone(), - )); + while let Some(incoming) = endpoint.accept().await { + let remote_addr: SocketAddr = incoming.remote_address(); + let connecting = incoming.accept(); + match connecting { + Ok(connecting) => { + tokio::task::spawn(handle_connecting_task( + endpoint.clone(), + connecting, + sender.clone(), + bank_forks.clone(), + prune_cache_pending.clone(), + router.clone(), + cache.clone(), + stats.clone(), + )); + } + Err(error) => { + debug!( + "Error while accepting incoming connection: {error:?} from {}", + remote_addr + ); + } + } } report_metrics_task.abort(); } @@ -629,11 +652,15 @@ struct TurbineQuicStats { connection_error_timed_out: AtomicU64, connection_error_transport_error: AtomicU64, connection_error_version_mismatch: AtomicU64, + connection_error_connection_limit_exceeded: AtomicU64, invalid_identity: AtomicU64, router_try_send_error_full: AtomicU64, send_datagram_error_connection_lost: AtomicU64, send_datagram_error_too_large: AtomicU64, send_datagram_error_unsupported_by_peer: AtomicU64, + connect_error_cids_exhausted: AtomicU64, + connect_error_invalid_server_name: AtomicU64, + connection_error_cids_exhausted: AtomicU64, } async fn report_metrics_task(name: &'static str, stats: Arc) { @@ -649,12 +676,6 @@ fn record_error(err: &Error, stats: &TurbineQuicStats) { Error::ConnectError(ConnectError::EndpointStopping) => { add_metric!(stats.connect_error_other) } - Error::ConnectError(ConnectError::TooManyConnections) => { - add_metric!(stats.connect_error_too_many_connections) - } - Error::ConnectError(ConnectError::InvalidDnsName(_)) => { - add_metric!(stats.connect_error_other) - } Error::ConnectError(ConnectError::InvalidRemoteAddress(_)) => { add_metric!(stats.connect_error_invalid_remote_address) } @@ -696,6 +717,15 @@ fn record_error(err: &Error, stats: &TurbineQuicStats) { add_metric!(stats.send_datagram_error_connection_lost) } Error::TlsError(_) => (), + Error::ConnectError(ConnectError::CidsExhausted) => { + add_metric!(stats.connect_error_cids_exhausted) + } + Error::ConnectError(ConnectError::InvalidServerName(_)) => { + add_metric!(stats.connect_error_invalid_server_name) + } + Error::ConnectionError(ConnectionError::CidsExhausted) => { + add_metric!(stats.connection_error_cids_exhausted) + } } } @@ -757,6 +787,11 @@ fn report_metrics(name: &'static str, stats: &TurbineQuicStats) { reset_metric!(stats.connection_error_version_mismatch), i64 ), + ( + "connection_error_connection_limit_exceeded", + reset_metric!(stats.connection_error_connection_limit_exceeded), + i64 + ), ( "invalid_identity", reset_metric!(stats.invalid_identity),