From 2d474172c46dd8b56a6b456298b772f27bc98c76 Mon Sep 17 00:00:00 2001 From: joii Date: Thu, 23 May 2024 14:13:37 +0800 Subject: [PATCH 1/4] Add features: fuzz --- network/Cargo.toml | 1 + network/src/peer_store/peer_store_impl.rs | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/network/Cargo.toml b/network/Cargo.toml index a3c85d0ff5..46afbe2627 100644 --- a/network/Cargo.toml +++ b/network/Cargo.toml @@ -41,6 +41,7 @@ p2p = { version="0.4.0", package="tentacle", features = ["upnp", "parking_lot", [features] with_sentry = ["sentry"] with_dns_seeding = ["lazy_static", "bs58", "faster-hex", "trust-dns-resolver", "secp256k1"] +fuzz = [] [dev-dependencies] tempfile.workspace = true diff --git a/network/src/peer_store/peer_store_impl.rs b/network/src/peer_store/peer_store_impl.rs index f7e39c3cd8..4c076066a2 100644 --- a/network/src/peer_store/peer_store_impl.rs +++ b/network/src/peer_store/peer_store_impl.rs @@ -71,6 +71,26 @@ impl PeerStore { Ok(()) } + #[cfg(feature = "fuzz")] + pub fn add_addr_fuzz( + &mut self, + addr: Multiaddr, + flags: Flags, + last_connected_at_ms: u64, + attempts_count: u32, + ) -> Result<()> { + if self.ban_list.is_addr_banned(&addr) { + return Ok(()); + } + self.check_purge()?; + let score = self.score_config.default_score; + let mut addr_info = AddrInfo::new(addr, last_connected_at_ms, score, flags.bits()); + addr_info.attempts_count = attempts_count; + + self.addr_manager.add(addr_info); + Ok(()) + } + /// Add outbound peer address pub fn add_outbound_addr(&mut self, addr: Multiaddr, flags: Flags) { if self.ban_list.is_addr_banned(&addr) { From d8ada11ba686a9a54523132cd9967625ae66db18 Mon Sep 17 00:00:00 2001 From: joii Date: Thu, 23 May 2024 14:14:20 +0800 Subject: [PATCH 2/4] Add fuzz on network --- network/fuzz/.gitignore | 4 + network/fuzz/Cargo.toml | 49 ++++++ .../fuzz/fuzz_targets/fuzz_addr_manager.rs | 98 ++++++++++++ network/fuzz/fuzz_targets/fuzz_compress.rs | 14 ++ network/fuzz/fuzz_targets/fuzz_decompress.rs | 11 ++ network/fuzz/fuzz_targets/fuzz_peer_store.rs | 117 ++++++++++++++ network/fuzz/rust-toolchain | 1 + network/fuzz/src/lib.rs | 143 ++++++++++++++++++ 8 files changed, 437 insertions(+) create mode 100644 network/fuzz/.gitignore create mode 100644 network/fuzz/Cargo.toml create mode 100644 network/fuzz/fuzz_targets/fuzz_addr_manager.rs create mode 100644 network/fuzz/fuzz_targets/fuzz_compress.rs create mode 100644 network/fuzz/fuzz_targets/fuzz_decompress.rs create mode 100644 network/fuzz/fuzz_targets/fuzz_peer_store.rs create mode 100644 network/fuzz/rust-toolchain create mode 100644 network/fuzz/src/lib.rs diff --git a/network/fuzz/.gitignore b/network/fuzz/.gitignore new file mode 100644 index 0000000000..1a45eee776 --- /dev/null +++ b/network/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/network/fuzz/Cargo.toml b/network/fuzz/Cargo.toml new file mode 100644 index 0000000000..3b438562c2 --- /dev/null +++ b/network/fuzz/Cargo.toml @@ -0,0 +1,49 @@ +[package] +name = "ckb-network-fuzz" +version = "0.0.0" +publish = false +edition = "2021" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +rand = "0.7" +ipnetwork = "0.18" +ckb-systemtime = { path = "../../util/systemtime/" } + +[dependencies.ckb-network] +path = ".." +features = ["fuzz"] + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[profile.release] +debug = 1 + +[[bin]] +name = "fuzz_compress" +path = "fuzz_targets/fuzz_compress.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_decompress" +path = "fuzz_targets/fuzz_decompress.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_addr_manager" +path = "fuzz_targets/fuzz_addr_manager.rs" +test = false +doc = false + +[[bin]] +name = "fuzz_peer_store" +path = "fuzz_targets/fuzz_peer_store.rs" +test = false +doc = false diff --git a/network/fuzz/fuzz_targets/fuzz_addr_manager.rs b/network/fuzz/fuzz_targets/fuzz_addr_manager.rs new file mode 100644 index 0000000000..f4665c0512 --- /dev/null +++ b/network/fuzz/fuzz_targets/fuzz_addr_manager.rs @@ -0,0 +1,98 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +use ckb_network::{ + peer_store::{addr_manager::AddrManager, types::AddrInfo}, + PeerId, +}; + +use std::collections::HashSet; +use std::net::Ipv4Addr; + +const ADDR_SIZE: usize = 4 + 32; // IPv4+Port+SHA256 + +fn new_addr(data: &[u8], index: usize) -> AddrInfo { + let pos = 8 + index * ADDR_SIZE; + let data = data[pos..pos + ADDR_SIZE].to_vec(); + + let ip = Ipv4Addr::from(u32::from_le_bytes(data[0..4].try_into().unwrap())); + // let ip = Ipv4Addr::from(((225 << 24) + index) as u32); + // let port = u16::from_le_bytes(data[4..6].try_into().unwrap()); + let peer_id = + PeerId::from_bytes(vec![vec![0x12], vec![0x20], data[4..].to_vec()].concat()).unwrap(); + + AddrInfo::new( + format!("/ip4/{}/tcp/43/p2p/{}", ip, peer_id.to_base58()) + .parse() + .unwrap(), + 0, + 0, + 0, + ) +} + +fn test_remove( + mut addr_manager: AddrManager, + basic: &HashSet, + rm_num: usize, +) -> AddrManager { + let removed = addr_manager.fetch_random(rm_num, |_| true); + // assert_eq!(removed.len(), rm_num.min(basic.len())); + assert!(removed.len() <= rm_num.min(basic.len())); + + for addr in &removed { + addr_manager.remove(&addr.addr); + } + assert!(addr_manager.count() <= (basic.len() - removed.len())); + for addr in removed { + addr_manager.add(addr); + } + assert!(addr_manager.count() <= basic.len()); + + addr_manager +} + +fuzz_target!(|data: &[u8]| { + if data.len() < 8 + ADDR_SIZE { + return; + } + if (data.len() - 8) % ADDR_SIZE != 0 { + return; + } + let scale: u16 = u16::from_le_bytes(data[0..2].try_into().unwrap()) % 100; + let (basic_len, added_len) = { + let t = (data.len() - 8) / ADDR_SIZE; + + let b = (t as f32 / 100.0 * scale as f32) as usize; + (b, t - b) + }; + let mut addr_manager = AddrManager::default(); + + let mut basic = HashSet::new(); + for i in 0..basic_len { + let addr = new_addr(data, i); + basic.insert(addr.clone()); + addr_manager.add(addr); + } + assert!(basic.len() >= addr_manager.count()); + + let removed_num1 = + u16::from_le_bytes(data[2..4].try_into().unwrap()) as usize % (basic.len() + 8); + let mut addr_manager = test_remove(addr_manager, &basic, removed_num1); + + let mut added = Vec::new(); + for i in 0..added_len { + let addr = new_addr(data, i + basic_len); + added.push(addr.clone()); + + addr_manager.add(addr.clone()); + basic.insert(addr); + } + assert!(basic.len() >= addr_manager.count()); + + let removed_num2 = + u16::from_le_bytes(data[4..6].try_into().unwrap()) as usize % (basic.len() + 4); + + let mut _addr_manager = test_remove(addr_manager, &basic, removed_num2); +}); diff --git a/network/fuzz/fuzz_targets/fuzz_compress.rs b/network/fuzz/fuzz_targets/fuzz_compress.rs new file mode 100644 index 0000000000..76ee180d7e --- /dev/null +++ b/network/fuzz/fuzz_targets/fuzz_compress.rs @@ -0,0 +1,14 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +use ckb_network::bytes::{Bytes, BytesMut}; +use ckb_network::compress::{compress, decompress}; + +fuzz_target!(|data: &[u8]| { + let raw_data = Bytes::from(data.to_vec()); + + let cmp_data = compress(raw_data.clone()); + let demsg = decompress(BytesMut::from(cmp_data.as_ref())).unwrap(); + assert_eq!(raw_data, demsg); +}); diff --git a/network/fuzz/fuzz_targets/fuzz_decompress.rs b/network/fuzz/fuzz_targets/fuzz_decompress.rs new file mode 100644 index 0000000000..1a151e299d --- /dev/null +++ b/network/fuzz/fuzz_targets/fuzz_decompress.rs @@ -0,0 +1,11 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +use ckb_network::bytes::{Bytes, BytesMut}; +use ckb_network::compress::decompress; + +fuzz_target!(|data: &[u8]| { + let raw_data = Bytes::from(data.to_vec()); + let _demsg = decompress(BytesMut::from(raw_data.as_ref())); +}); diff --git a/network/fuzz/fuzz_targets/fuzz_peer_store.rs b/network/fuzz/fuzz_targets/fuzz_peer_store.rs new file mode 100644 index 0000000000..b26e681366 --- /dev/null +++ b/network/fuzz/fuzz_targets/fuzz_peer_store.rs @@ -0,0 +1,117 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +use ckb_network::{ + multiaddr::MultiAddr, peer_store::types::BannedAddr, peer_store::PeerStore, Flags, PeerId, +}; +use ckb_network_fuzz::BufManager; +use rand::{thread_rng, RngCore}; + +fn new_multi_addr(data: &mut BufManager) -> (MultiAddr, Flags) { + let flags = data.get(); + let addr_flag = data.get::(); + + let mut addr_str = if addr_flag & 0b1 != 1 { + let buf = data.get_buf(16); + format!( + "/ip6/{}", + std::net::Ipv6Addr::from(u128::from_le_bytes(buf.try_into().unwrap())).to_string() + ) + } else { + format!("/ip4/{}", data.get::().to_string()) + }; + + addr_str += &format!("/tcp/{}", data.get::()); + + addr_str += &format!("/p2p/{}", data.get::().to_base58()); + + (addr_str.parse().unwrap(), flags) +} + +fn add_ban_addr(data: &mut BufManager, peer_store: &mut PeerStore) { + let now_ms = ckb_systemtime::unix_time_as_millis(); + + let num = data.get::() as usize; + for _ in 0..num { + let flags = data.get::(); + + let network = if flags & 0b1 == 1 { + data.get::().into() + } else { + data.get::().into() + }; + + let ban_addr = BannedAddr { + address: network, + ban_until: now_ms + (data.get::() as u64), + created_at: now_ms, + ban_reason: String::new(), + }; + peer_store.mut_ban_list().ban(ban_addr); + } +} + +fn add_basic_addr(data: &mut BufManager, peer_store: &mut PeerStore) { + let flags = data.get::(); + if flags & 0b1 == 0 { + return; + } + + if (flags >> 1) & 0b1 == 1 { + add_ban_addr(data, peer_store); + } + + let basic_num = data.get::(); + + let last_connected_time_range = if flags >> 2 & 0b1 == 1 { + 60_000 // 1 minute + } else { + 1000 * 60 * 60 * 24 * 365 + }; + + let num = basic_num % 16 + (16384) - 8; // ±8 + let mut rng = thread_rng(); + let now_ms = ckb_systemtime::unix_time_as_millis(); + + for i in 0..num { + let addr = format!( + "/ip4/{}/tcp/43/p2p/{}", + std::net::Ipv4Addr::from(i as u32).to_string(), + PeerId::random().to_base58() + ) + .parse() + .unwrap(); + let _ = peer_store.add_addr_fuzz( + addr, + Flags::all(), + now_ms - ((rng.next_u32() as u64) % last_connected_time_range), + rng.next_u32(), + ); + } +} + +fuzz_target!(|data: &[u8]| { + let mut data = BufManager::new(data); + + let now_ms = ckb_systemtime::unix_time_as_millis(); + + let mut peer_store: PeerStore = Default::default(); + + // basic addr: + add_basic_addr(&mut data, &mut peer_store); + + let fetch_count = data.get::() as usize; + let fetch_flag = data.get(); + + while !data.is_end() { + let (addr, flag) = new_multi_addr(&mut data); + let last_connected_time = now_ms + data.get::() as u64; + let attempts_count = data.get::(); + let _res = peer_store.add_addr_fuzz(addr, flag, last_connected_time, attempts_count); + // _res.expect("msg"); + } + + let ret = peer_store.fetch_random_addrs(fetch_count, fetch_flag); + assert!(ret.len() <= fetch_count); +}); diff --git a/network/fuzz/rust-toolchain b/network/fuzz/rust-toolchain new file mode 100644 index 0000000000..e426a2f8a7 --- /dev/null +++ b/network/fuzz/rust-toolchain @@ -0,0 +1 @@ +nightly-2024-05-14 diff --git a/network/fuzz/src/lib.rs b/network/fuzz/src/lib.rs new file mode 100644 index 0000000000..94138a6b27 --- /dev/null +++ b/network/fuzz/src/lib.rs @@ -0,0 +1,143 @@ +use ckb_network::Flags; +use ckb_network::PeerId; + +pub struct BufManager<'a> { + buf: &'a [u8], + offset: usize, +} + +impl<'a> BufManager<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { + buf: data, + offset: 0, + } + } + + pub fn len(&self) -> usize { + self.buf.len() + } + + pub fn get_buf(&mut self, len: usize) -> Vec { + let buf_len = self.buf.len(); + if buf_len >= self.offset + len && self.offset != buf_len { + let r = self.buf[self.offset..self.offset + len].to_vec(); + self.offset += len; + r + } else { + let mut r = Vec::::with_capacity(len); + r.resize(len, 0); + r[0..(buf_len - self.offset)].copy_from_slice(&self.buf[self.offset..]); + self.offset = buf_len; + r + } + } + + pub fn get>(&mut self) -> T { + T::from_bytes(&self.get_buf(T::type_size())) + } + + pub fn is_end(&self) -> bool { + self.offset >= self.buf.len() + } +} + +pub trait FromBytes { + fn from_bytes(d: &[u8]) -> T; + fn type_size() -> usize; +} + +impl FromBytes for u8 { + fn type_size() -> usize { + 1 + } + fn from_bytes(d: &[u8]) -> u8 { + u8::from_le_bytes(d.try_into().unwrap()) + } +} +impl FromBytes for u16 { + fn type_size() -> usize { + 2 + } + fn from_bytes(d: &[u8]) -> u16 { + u16::from_le_bytes(d.try_into().unwrap()) + } +} +impl FromBytes for u32 { + fn type_size() -> usize { + 4 + } + fn from_bytes(d: &[u8]) -> u32 { + u32::from_le_bytes(d.try_into().unwrap()) + } +} +impl FromBytes for u64 { + fn type_size() -> usize { + 8 + } + fn from_bytes(d: &[u8]) -> u64 { + u64::from_le_bytes(d.try_into().unwrap()) + } +} +impl FromBytes for u128 { + fn type_size() -> usize { + 8 + } + fn from_bytes(d: &[u8]) -> u128 { + u128::from_le_bytes(d.try_into().unwrap()) + } +} +impl FromBytes for Flags { + fn type_size() -> usize { + 1 + } + fn from_bytes(d: &[u8]) -> Flags { + unsafe { + Flags::from_bits_unchecked( + (u8::from_le_bytes(d.try_into().unwrap()) % 0b1000000) as u64, + ) + } + } +} + +impl FromBytes for std::net::Ipv4Addr { + fn type_size() -> usize { + 4 + } + fn from_bytes(d: &[u8]) -> std::net::Ipv4Addr { + std::net::Ipv4Addr::from(u32::from_bytes(d)) + } +} +impl FromBytes for std::net::Ipv6Addr { + fn type_size() -> usize { + 16 + } + fn from_bytes(d: &[u8]) -> std::net::Ipv6Addr { + std::net::Ipv6Addr::from(u128::from_bytes(d)) + } +} + +impl FromBytes for ipnetwork::Ipv4Network { + fn type_size() -> usize { + 4 + } + fn from_bytes(d: &[u8]) -> ipnetwork::Ipv4Network { + Self::from(std::net::Ipv4Addr::from_bytes(d)) + } +} +impl FromBytes for ipnetwork::Ipv6Network { + fn type_size() -> usize { + 16 + } + fn from_bytes(d: &[u8]) -> ipnetwork::Ipv6Network { + Self::from(std::net::Ipv6Addr::from_bytes(d)) + } +} +impl FromBytes for PeerId { + fn type_size() -> usize { + 32 + } + fn from_bytes(d: &[u8]) -> PeerId { + PeerId::from_bytes(vec![vec![0x12], vec![0x20], d.to_vec()].concat()).unwrap() + } +} From c216516518aefbf88819823287acc699a6796b4e Mon Sep 17 00:00:00 2001 From: joii Date: Thu, 23 May 2024 15:48:33 +0800 Subject: [PATCH 3/4] Fix CI: Improve the Cargo.toml of fuzz --- network/Cargo.toml | 1 + network/fuzz/Cargo.toml | 10 ++++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/network/Cargo.toml b/network/Cargo.toml index 46afbe2627..a0fa919f96 100644 --- a/network/Cargo.toml +++ b/network/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" description = "ckb network implementation" homepage = "https://github.com/nervosnetwork/ckb" repository = "https://github.com/nervosnetwork/ckb" +exclude = [ "fuzz" ] [dependencies] rand = "0.7" diff --git a/network/fuzz/Cargo.toml b/network/fuzz/Cargo.toml index 3b438562c2..744ee73155 100644 --- a/network/fuzz/Cargo.toml +++ b/network/fuzz/Cargo.toml @@ -1,8 +1,12 @@ [package] name = "ckb-network-fuzz" -version = "0.0.0" +version = "0.117.0-pre" publish = false edition = "2021" +license = "MIT" +description = "ckb network fuzz testing" +homepage = "https://github.com/nervosnetwork/ckb" +repository = "https://github.com/nervosnetwork/ckb" [package.metadata] cargo-fuzz = true @@ -11,7 +15,9 @@ cargo-fuzz = true libfuzzer-sys = "0.4" rand = "0.7" ipnetwork = "0.18" -ckb-systemtime = { path = "../../util/systemtime/" } + +[dependencies.ckb-systemtime] +path = "../../util/systemtime" [dependencies.ckb-network] path = ".." From cdc1c2d1945d02ea8b5aece6188a670f040d119e Mon Sep 17 00:00:00 2001 From: joii Date: Sat, 25 May 2024 13:05:28 +0800 Subject: [PATCH 4/4] Optimize code based on responses --- network/fuzz/Cargo.toml | 4 --- network/fuzz/fuzz_targets/fuzz_peer_store.rs | 26 +++----------------- network/fuzz/rust-toolchain | 2 +- 3 files changed, 5 insertions(+), 27 deletions(-) diff --git a/network/fuzz/Cargo.toml b/network/fuzz/Cargo.toml index 744ee73155..f29a3f976a 100644 --- a/network/fuzz/Cargo.toml +++ b/network/fuzz/Cargo.toml @@ -13,12 +13,8 @@ cargo-fuzz = true [dependencies] libfuzzer-sys = "0.4" -rand = "0.7" ipnetwork = "0.18" -[dependencies.ckb-systemtime] -path = "../../util/systemtime" - [dependencies.ckb-network] path = ".." features = ["fuzz"] diff --git a/network/fuzz/fuzz_targets/fuzz_peer_store.rs b/network/fuzz/fuzz_targets/fuzz_peer_store.rs index b26e681366..255974f715 100644 --- a/network/fuzz/fuzz_targets/fuzz_peer_store.rs +++ b/network/fuzz/fuzz_targets/fuzz_peer_store.rs @@ -6,7 +6,6 @@ use ckb_network::{ multiaddr::MultiAddr, peer_store::types::BannedAddr, peer_store::PeerStore, Flags, PeerId, }; use ckb_network_fuzz::BufManager; -use rand::{thread_rng, RngCore}; fn new_multi_addr(data: &mut BufManager) -> (MultiAddr, Flags) { let flags = data.get(); @@ -30,8 +29,6 @@ fn new_multi_addr(data: &mut BufManager) -> (MultiAddr, Flags) { } fn add_ban_addr(data: &mut BufManager, peer_store: &mut PeerStore) { - let now_ms = ckb_systemtime::unix_time_as_millis(); - let num = data.get::() as usize; for _ in 0..num { let flags = data.get::(); @@ -44,8 +41,8 @@ fn add_ban_addr(data: &mut BufManager, peer_store: &mut PeerStore) { let ban_addr = BannedAddr { address: network, - ban_until: now_ms + (data.get::() as u64), - created_at: now_ms, + ban_until: data.get(), + created_at: data.get(), ban_reason: String::new(), }; peer_store.mut_ban_list().ban(ban_addr); @@ -64,15 +61,7 @@ fn add_basic_addr(data: &mut BufManager, peer_store: &mut PeerStore) { let basic_num = data.get::(); - let last_connected_time_range = if flags >> 2 & 0b1 == 1 { - 60_000 // 1 minute - } else { - 1000 * 60 * 60 * 24 * 365 - }; - let num = basic_num % 16 + (16384) - 8; // ±8 - let mut rng = thread_rng(); - let now_ms = ckb_systemtime::unix_time_as_millis(); for i in 0..num { let addr = format!( @@ -82,20 +71,13 @@ fn add_basic_addr(data: &mut BufManager, peer_store: &mut PeerStore) { ) .parse() .unwrap(); - let _ = peer_store.add_addr_fuzz( - addr, - Flags::all(), - now_ms - ((rng.next_u32() as u64) % last_connected_time_range), - rng.next_u32(), - ); + let _ = peer_store.add_addr_fuzz(addr, Flags::all(), data.get(), data.get()); } } fuzz_target!(|data: &[u8]| { let mut data = BufManager::new(data); - let now_ms = ckb_systemtime::unix_time_as_millis(); - let mut peer_store: PeerStore = Default::default(); // basic addr: @@ -106,7 +88,7 @@ fuzz_target!(|data: &[u8]| { while !data.is_end() { let (addr, flag) = new_multi_addr(&mut data); - let last_connected_time = now_ms + data.get::() as u64; + let last_connected_time = data.get(); let attempts_count = data.get::(); let _res = peer_store.add_addr_fuzz(addr, flag, last_connected_time, attempts_count); // _res.expect("msg"); diff --git a/network/fuzz/rust-toolchain b/network/fuzz/rust-toolchain index e426a2f8a7..07ade694b1 100644 --- a/network/fuzz/rust-toolchain +++ b/network/fuzz/rust-toolchain @@ -1 +1 @@ -nightly-2024-05-14 +nightly \ No newline at end of file