From afbfef20fb78921cc6dc365878fb5b3a790ee769 Mon Sep 17 00:00:00 2001 From: acerone85 Date: Thu, 14 Nov 2024 11:39:32 +0000 Subject: [PATCH] Refactor fuel-core to use version of StorageRead::read with offset --- Cargo.lock | 27 +++++++------------ Cargo.toml | 3 +++ crates/fuel-core/src/graphql_api/database.rs | 9 +++++-- crates/fuel-core/src/state/data_source.rs | 3 ++- .../fuel-core/src/state/generic_database.rs | 12 ++++++--- .../fuel-core/src/state/historical_rocksdb.rs | 4 ++- .../src/state/iterable_key_value_view.rs | 3 ++- crates/fuel-core/src/state/key_value_view.rs | 3 ++- crates/fuel-core/src/state/rocks_db.rs | 15 ++++++++--- crates/services/txpool_v2/src/tests/mocks.rs | 16 +++++++---- crates/storage/src/kv_store.rs | 23 ++++++++++------ crates/storage/src/structured_storage.rs | 12 ++++++--- crates/storage/src/transactional.rs | 3 ++- crates/storage/src/vm_storage.rs | 9 +++++-- 14 files changed, 92 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 67a97f72c3e..c76eae062ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3166,8 +3166,7 @@ dependencies = [ [[package]] name = "fuel-asm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f325971bf9047ec70004f80a989e03456316bc19cbef3ff3a39a38b192ab56e" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "bitflags 2.6.0", "fuel-types 0.58.2", @@ -3178,8 +3177,7 @@ dependencies = [ [[package]] name = "fuel-compression" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24e42841f56f76ed759b3f516e5188d5c42de47015bee951651660c13b6dfa6c" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "fuel-derive 0.58.2", "fuel-types 0.58.2", @@ -3893,8 +3891,7 @@ dependencies = [ [[package]] name = "fuel-crypto" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e318850ca64890ff123a99b6b866954ef49da94ab9bc6827cf6ee045568585" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "coins-bip32", "coins-bip39", @@ -3926,8 +3923,7 @@ dependencies = [ [[package]] name = "fuel-derive" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab0bc46a3552964bae5169e79b383761a54bd115ea66951a1a7a229edcefa55a" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "proc-macro2", "quote", @@ -3963,8 +3959,7 @@ dependencies = [ [[package]] name = "fuel-merkle" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79eca6a452311c70978a5df796c0f99f27e474b69719e0db4c1d82e68800d07" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "derive_more 0.99.18", "digest 0.10.7", @@ -3984,8 +3979,7 @@ checksum = "4c1b711f28553ddc5f3546711bd220e144ce4c1af7d9e9a1f70b2f20d9f5b791" [[package]] name = "fuel-storage" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d0c46b5d76b3e11197bd31e036cd8b1cb46c4d822cacc48836638080c6d2b76" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" [[package]] name = "fuel-tx" @@ -4012,8 +4006,7 @@ dependencies = [ [[package]] name = "fuel-tx" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6723bb8710ba2b70516ac94d34459593225870c937670fb3afaf82e0354667ac" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "bitflags 2.6.0", "derivative", @@ -4046,8 +4039,7 @@ dependencies = [ [[package]] name = "fuel-types" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982265415a99b5bd6277bc24194a233bb2e18764df11c937b3dbb11a02c9e545" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "fuel-derive 0.58.2", "hex", @@ -4089,8 +4081,7 @@ dependencies = [ [[package]] name = "fuel-vm" version = "0.58.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b5362d7d072c72eec20581f67fc5400090c356a7f3ae77c79880b3b177b667" +source = "git+https://github.com/fuelLabs/fuel-vm#486d06f83de6ea3411543f3034700b0157a65eb5" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d80bf4f5bdc..27dc602e521 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -143,3 +143,6 @@ itertools = { version = "0.12", default-features = false } insta = "1.8" tempfile = "3.4" tikv-jemallocator = "0.5" + +[patch.crates-io] +fuel-vm = { git = "https://github.com/fuelLabs/fuel-vm" } diff --git a/crates/fuel-core/src/graphql_api/database.rs b/crates/fuel-core/src/graphql_api/database.rs index bf47c8d92a7..0c918e35244 100644 --- a/crates/fuel-core/src/graphql_api/database.rs +++ b/crates/fuel-core/src/graphql_api/database.rs @@ -252,8 +252,13 @@ impl StorageSize for ReadView { } impl StorageRead for ReadView { - fn read(&self, key: &BlobId, buf: &mut [u8]) -> Result, Self::Error> { - StorageRead::::read(self.on_chain.as_ref(), key, buf) + fn read( + &self, + key: &BlobId, + offset: usize, + buf: &mut [u8], + ) -> Result, Self::Error> { + StorageRead::::read(self.on_chain.as_ref(), key, offset, buf) } fn read_alloc(&self, key: &BlobId) -> Result>, Self::Error> { diff --git a/crates/fuel-core/src/state/data_source.rs b/crates/fuel-core/src/state/data_source.rs index cf107d0e1b9..f30f3ee8268 100644 --- a/crates/fuel-core/src/state/data_source.rs +++ b/crates/fuel-core/src/state/data_source.rs @@ -67,9 +67,10 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.data.read(key, column, buf) + self.data.read(key, column, offset, buf) } } diff --git a/crates/fuel-core/src/state/generic_database.rs b/crates/fuel-core/src/state/generic_database.rs index 8306df1b1fb..dbead6446c5 100644 --- a/crates/fuel-core/src/state/generic_database.rs +++ b/crates/fuel-core/src/state/generic_database.rs @@ -79,8 +79,13 @@ where M: Mappable, StructuredStorage: StorageRead, { - fn read(&self, key: &M::Key, buf: &mut [u8]) -> Result, Self::Error> { - self.storage.storage::().read(key, buf) + fn read( + &self, + key: &M::Key, + offset: usize, + buf: &mut [u8], + ) -> Result, Self::Error> { + self.storage.storage::().read(key, offset, buf) } fn read_alloc(&self, key: &M::Key) -> Result>, Self::Error> { @@ -124,9 +129,10 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - KeyValueInspect::read(&self.storage, key, column, buf) + KeyValueInspect::read(&self.storage, key, column, offset, buf) } } diff --git a/crates/fuel-core/src/state/historical_rocksdb.rs b/crates/fuel-core/src/state/historical_rocksdb.rs index 52887fdaf2e..c43fefa8209 100644 --- a/crates/fuel-core/src/state/historical_rocksdb.rs +++ b/crates/fuel-core/src/state/historical_rocksdb.rs @@ -398,9 +398,11 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.db.read(key, Column::OriginalColumn(column), buf) + self.db + .read(key, Column::OriginalColumn(column), offset, buf) } } diff --git a/crates/fuel-core/src/state/iterable_key_value_view.rs b/crates/fuel-core/src/state/iterable_key_value_view.rs index 40ab3483ede..1d8fd02cbbb 100644 --- a/crates/fuel-core/src/state/iterable_key_value_view.rs +++ b/crates/fuel-core/src/state/iterable_key_value_view.rs @@ -67,9 +67,10 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.0.read(key, column, buf) + self.0.read(key, column, offset, buf) } } diff --git a/crates/fuel-core/src/state/key_value_view.rs b/crates/fuel-core/src/state/key_value_view.rs index 9e70037fc21..e507e83329a 100644 --- a/crates/fuel-core/src/state/key_value_view.rs +++ b/crates/fuel-core/src/state/key_value_view.rs @@ -54,8 +54,9 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.0.read(key, column, buf) + self.0.read(key, column, offset, buf) } } diff --git a/crates/fuel-core/src/state/rocks_db.rs b/crates/fuel-core/src/state/rocks_db.rs index c751ebe7c55..5fa5855ced6 100644 --- a/crates/fuel-core/src/state/rocks_db.rs +++ b/crates/fuel-core/src/state/rocks_db.rs @@ -28,6 +28,7 @@ use fuel_core_storage::{ WriteOperation, }, transactional::Changes, + Error as StorageError, Result as StorageResult, }; use itertools::Itertools; @@ -732,6 +733,7 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, mut buf: &mut [u8], ) -> StorageResult> { self.metrics.read_meter.inc(); @@ -743,10 +745,15 @@ where .get_pinned_cf_opt(&self.cf(column), key, &self.read_options) .map_err(|e| DatabaseError::Other(e.into()))? .map(|value| { - let read = value.len(); - std::io::Write::write_all(&mut buf, value.as_ref()) - .map_err(|e| DatabaseError::Other(anyhow::anyhow!(e)))?; - StorageResult::Ok(read) + if let Some(read) = value.len().checked_sub(offset) { + std::io::Write::write_all(&mut buf, value[offset..].as_ref()) + .map_err(|e| DatabaseError::Other(anyhow::anyhow!(e)))?; + StorageResult::Ok(read) + } else { + Err(StorageError::Other(anyhow::anyhow!( + "Offset is out of bounds" + ))) + } }) .transpose()?; diff --git a/crates/services/txpool_v2/src/tests/mocks.rs b/crates/services/txpool_v2/src/tests/mocks.rs index 1dca65bc9d3..88dd337bd66 100644 --- a/crates/services/txpool_v2/src/tests/mocks.rs +++ b/crates/services/txpool_v2/src/tests/mocks.rs @@ -126,16 +126,22 @@ impl StorageRead for MockDb { fn read( &self, key: &::Key, + offset: usize, buf: &mut [u8], ) -> Result, Self::Error> { let table = self.data.lock().unwrap(); let bytes = table.blobs.get(key); - let len = bytes.map(|bytes| { - buf.copy_from_slice(bytes.0.as_slice()); - bytes.0.len() - }); - Ok(len) + bytes + .map(|bytes| { + if let Some(len) = bytes.0.len().checked_sub(offset) { + buf.copy_from_slice(&bytes.0[offset..]); + Ok(len) + } else { + Err(()) + } + }) + .transpose() } fn read_alloc( diff --git a/crates/storage/src/kv_store.rs b/crates/storage/src/kv_store.rs index 67ab9493c7f..041716eb4b3 100644 --- a/crates/storage/src/kv_store.rs +++ b/crates/storage/src/kv_store.rs @@ -67,18 +67,24 @@ pub trait KeyValueInspect { &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { self.get(key, column)? .map(|value| { - let read = value.len(); - if read != buf.len() { - return Err(StorageError::Other(anyhow::anyhow!( - "Buffer size is not equal to the value size" - ))); + if let Some(read) = value.len().checked_sub(offset) { + if read != buf.len() { + return Err(StorageError::Other(anyhow::anyhow!( + "Buffer size is not equal to the value size" + ))); + }; + buf.copy_from_slice(value[offset..].as_ref()); + Ok(buf.len()) + } else { + Err(StorageError::Other(anyhow::anyhow!( + "Offset is out of bounds" + ))) } - buf.copy_from_slice(value.as_ref()); - Ok(read) }) .transpose() } @@ -111,9 +117,10 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.deref().read(key, column, buf) + self.deref().read(key, column, offset, buf) } } diff --git a/crates/storage/src/structured_storage.rs b/crates/storage/src/structured_storage.rs index e78e9637484..9de758d6c24 100644 --- a/crates/storage/src/structured_storage.rs +++ b/crates/storage/src/structured_storage.rs @@ -148,9 +148,10 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { - self.inner.read(key, column, buf) + self.inner.read(key, column, offset, buf) } } @@ -359,6 +360,7 @@ where fn read( &self, key: &::Key, + offset: usize, buf: &mut [u8], ) -> Result, Self::Error> { let key_encoder = @@ -366,8 +368,12 @@ where key, ); let key_bytes = key_encoder.as_bytes(); - self.inner - .read(key_bytes.as_ref(), ::column(), buf) + self.inner.read( + key_bytes.as_ref(), + ::column(), + offset, + buf, + ) } fn read_alloc( diff --git a/crates/storage/src/transactional.rs b/crates/storage/src/transactional.rs index 14ec74159ed..4dd1a1782bb 100644 --- a/crates/storage/src/transactional.rs +++ b/crates/storage/src/transactional.rs @@ -388,6 +388,7 @@ where &self, key: &[u8], column: Self::Column, + offset: usize, buf: &mut [u8], ) -> StorageResult> { if let Some(operation) = self.get_from_changes(key, column) { @@ -405,7 +406,7 @@ where WriteOperation::Remove => Ok(None), } } else { - self.storage.read(key, column, buf) + self.storage.read(key, column, offset, buf) } } } diff --git a/crates/storage/src/vm_storage.rs b/crates/storage/src/vm_storage.rs index 92ab65ffdaf..99f6c360628 100644 --- a/crates/storage/src/vm_storage.rs +++ b/crates/storage/src/vm_storage.rs @@ -190,8 +190,13 @@ impl StorageRead for VmStorage where D: StorageRead, { - fn read(&self, key: &M::Key, buf: &mut [u8]) -> Result, Self::Error> { - StorageRead::::read(&self.database, key, buf) + fn read( + &self, + key: &M::Key, + offset: usize, + buf: &mut [u8], + ) -> Result, Self::Error> { + StorageRead::::read(&self.database, key, offset, buf) } fn read_alloc(