diff --git a/node/src/chain_spec/crab.rs b/node/src/chain_spec/crab.rs index 88eacdf4..f426e311 100644 --- a/node/src/chain_spec/crab.rs +++ b/node/src/chain_spec/crab.rs @@ -138,9 +138,7 @@ pub fn local_config() -> ChainSpec { pub fn genesis_config() -> ChainSpec { // TODO: update this before final release ChainSpec::from_genesis( - // Name "Crab2", - // ID "crab2", ChainType::Live, move || { diff --git a/node/src/chain_spec/darwinia.rs b/node/src/chain_spec/darwinia.rs index e8204b74..e35d3156 100644 --- a/node/src/chain_spec/darwinia.rs +++ b/node/src/chain_spec/darwinia.rs @@ -138,9 +138,7 @@ pub fn local_config() -> ChainSpec { pub fn genesis_config() -> ChainSpec { // TODO: update this before final release ChainSpec::from_genesis( - // Name "Darwinia2", - // ID "darwinia2", ChainType::Live, move || { diff --git a/node/src/chain_spec/pangolin.rs b/node/src/chain_spec/pangolin.rs index 25d1239b..c8ddf4d0 100644 --- a/node/src/chain_spec/pangolin.rs +++ b/node/src/chain_spec/pangolin.rs @@ -138,9 +138,7 @@ pub fn local_config() -> ChainSpec { pub fn genesis_config() -> ChainSpec { // TODO: update this before final release ChainSpec::from_genesis( - // Name "Pangolin2", - // ID "pangolin2", ChainType::Live, move || { diff --git a/pallet/account-migration/src/lib.rs b/pallet/account-migration/src/lib.rs index c558bf0d..f39b1cf5 100644 --- a/pallet/account-migration/src/lib.rs +++ b/pallet/account-migration/src/lib.rs @@ -145,20 +145,12 @@ pub mod pallet { #[pallet::getter(fn deposit_of)] pub type Deposits = StorageMap<_, Blake2_128Concat, AccountId32, Vec>; - /// [`darwinia_staking::Bonded`] data. - /// - /// - #[pallet::storage] - #[pallet::getter(fn bonded)] - pub type Bonded = StorageMap<_, Twox64Concat, AccountId32, AccountId32>; - /// [`darwinia_staking::Ledgers`] data. #[pallet::storage] #[pallet::getter(fn ledger_of)] pub type Ledgers = StorageMap<_, Blake2_128Concat, AccountId32, Ledger>; // TODO: identity storages - // TODO: proxy storages #[pallet::call] impl Pallet { @@ -189,7 +181,7 @@ pub mod pallet { a, ); } - if let Some(v) = >::get(&from) { + if let Some(v) = >::take(&from) { let locked = v.iter().map(|v| v.locked()).sum(); >::insert( @@ -203,8 +195,8 @@ pub mod pallet { // https://github.dev/paritytech/substrate/blob/19162e43be45817b44c7d48e50d03f074f60fbf4/frame/vesting/src/lib.rs#L86 >::set_lock(*b"vesting ", &to, locked, reasons); } - if let Some(l) = >::get(&from).and_then(>::get) { - if let Some(ds) = >::get(&from) { + if let Some(l) = >::take(&from) { + if let Some(ds) = >::take(&from) { as Currency<_>>::transfer( &to, &darwinia_deposit::account_id(), diff --git a/tool/state-processor/.gitignore b/tool/state-processor/.gitignore index a9a20e83..2988f7b7 100644 --- a/tool/state-processor/.gitignore +++ b/tool/state-processor/.gitignore @@ -11,5 +11,8 @@ target ## npm node_modules +# Data +data + # Test data test-data diff --git a/tool/state-processor/Cargo.lock b/tool/state-processor/Cargo.lock index 36390ca6..145cc807 100644 --- a/tool/state-processor/Cargo.lock +++ b/tool/state-processor/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.20" @@ -49,6 +55,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitvec" version = "1.0.1" @@ -80,6 +98,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "572f695136211188308f16ad2ca5c851a712c464060ae6974944458eb83880ba" + [[package]] name = "byte-slice-cast" version = "1.2.2" @@ -92,6 +116,15 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "cc" +version = "1.0.78" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +dependencies = [ + "jobserver", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -113,6 +146,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if", +] + [[package]] name = "crunchy" version = "0.2.2" @@ -152,6 +194,18 @@ dependencies = [ "termcolor", ] +[[package]] +name = "filetime" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys", +] + [[package]] name = "fixed-hash" version = "0.8.0" @@ -164,6 +218,25 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "flate2" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "form_urlencoded" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" +dependencies = [ + "percent-encoding", +] + [[package]] name = "funty" version = "2.0.0" @@ -224,6 +297,16 @@ dependencies = [ "quick-error", ] +[[package]] +name = "idna" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "impl-codec" version = "0.6.0" @@ -250,6 +333,24 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4217ad341ebadf8d8e724e264f13e593e0648f5b3e94b3896a5df283be015ecc" +[[package]] +name = "jobserver" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "libc" version = "0.2.138" @@ -271,6 +372,15 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +[[package]] +name = "miniz_oxide" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" +dependencies = [ + "adler", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -309,6 +419,12 @@ dependencies = [ "syn", ] +[[package]] +name = "percent-encoding" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -407,6 +523,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "redox_syscall" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.7.0" @@ -424,18 +549,55 @@ version = "0.6.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rustc-hex" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustls" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539a2bfe908f471bfa933876bd1eb6a19cf2176d375f82ef7f99530a40e48c2c" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "ryu" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4501abdff3ae82a1c1b477a17252eb69cee9e66eb915c1abaa4f44d873df9f09" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "serde" version = "1.0.150" @@ -478,6 +640,12 @@ dependencies = [ "digest", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "state-processor" version = "0.0.0" @@ -495,6 +663,9 @@ dependencies = [ "subhasher", "subspector", "substorager", + "tar", + "ureq", + "zstd", ] [[package]] @@ -551,6 +722,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b55807c0344e1e6c04d7c965f5289c39a8d94ae23ed5c0b57aabac549f871c6" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "termcolor" version = "1.1.3" @@ -589,6 +771,21 @@ dependencies = [ "crunchy", ] +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.10" @@ -627,12 +824,60 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unicode-bidi" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" + [[package]] name = "unicode-ident" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "733b5ad78377302af52c0dbcb2623d78fe50e4b3bf215948ff29e9ee031d8566" +dependencies = [ + "base64", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + [[package]] name = "version_check" version = "0.9.4" @@ -645,6 +890,89 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" + +[[package]] +name = "web-sys" +version = "0.3.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" +dependencies = [ + "webpki", +] + [[package]] name = "winapi" version = "0.3.9" @@ -676,6 +1004,63 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" + +[[package]] +name = "windows_i686_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" + +[[package]] +name = "windows_i686_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.42.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" + [[package]] name = "wyz" version = "0.5.1" @@ -684,3 +1069,41 @@ checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" dependencies = [ "tap", ] + +[[package]] +name = "xattr" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d1526bbe5aaeb5eb06885f4d987bcdfa5e23187055de9b83fe00156a821fabc" +dependencies = [ + "libc", +] + +[[package]] +name = "zstd" +version = "0.12.1+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c947d2adc84ff9a59f2e3c03b81aa4128acf28d6ad7d56273f7e8af14e47bea" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cf39f730b440bab43da8fb5faf5f254574462f73f260f85f7987f32154ff17" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.4+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fa202f2ef00074143e219d15b62ffc317d17cc33909feac471c044087cad7b0" +dependencies = [ + "cc", + "libc", +] diff --git a/tool/state-processor/Cargo.toml b/tool/state-processor/Cargo.toml index b12f4dcb..09c0b5b9 100644 --- a/tool/state-processor/Cargo.toml +++ b/tool/state-processor/Cargo.toml @@ -16,6 +16,9 @@ pretty_env_logger = { version = "0.4" } primitive-types = { version = "0.12.1", features = ["codec"] } serde = { version = "1.0" } serde_json = { version = "1.0" } +tar = { version = "0.4" } +ureq = { version = "2.6" } +zstd = { version = "0.12" } # hack-ink subhasher = { git = "https://github.com/hack-ink/subalfred" } diff --git a/tool/state-processor/src/main.rs b/tool/state-processor/src/main.rs index 3b8d0eeb..76f52e0b 100644 --- a/tool/state-processor/src/main.rs +++ b/tool/state-processor/src/main.rs @@ -1,392 +1,39 @@ -mod balances; -mod evm; -mod indices; -mod proxy; -mod staking; -mod system; -mod vesting; +// Processor components. +mod processor; +use processor::*; +mod util; +use util::*; +// Runtime configurations. mod adjust; use adjust::*; mod configuration; use configuration::*; - mod type_registry; use type_registry::*; +// Runtime pallets. +mod balances; +mod evm; +mod indices; +mod proxy; +mod staking; +mod system; +mod vesting; + #[cfg(test)] mod tests; -// std -use std::{ - env, - fs::File, - io::{Read, Write}, - marker::PhantomData, - mem, - sync::RwLock, -}; -// crates.io -use anyhow::Result; -use fxhash::FxHashMap; -use once_cell::sync::Lazy; -use parity_scale_codec::{Decode, Encode}; -use serde::de::DeserializeOwned; -// hack-ink -use subspector::ChainSpec; - -type Map = FxHashMap; - -static NOW: Lazy> = Lazy::new(|| RwLock::new(0)); +pub type StdResult = std::result::Result; +pub type Result = anyhow::Result; fn main() -> Result<()> { - env::set_var("RUST_LOG", "state_processor"); + std::env::set_var("RUST_LOG", "state_processor"); pretty_env_logger::init(); - >::new()?.process()?; + // >::new()?.process()?; + // >::new()?.process()?; + >::new()?.process()?; Ok(()) } - -struct Processor { - solo_state: State, - para_state: State<()>, - shell_state: State<()>, - shell_chain_spec: ChainSpec, -} -impl Processor -where - S: Configurable, -{ - fn new() -> Result { - let mut shell_chain_spec = - from_file::(&format!("test-data/{}-shell.json", S::NAME))?; - - Ok(Self { - solo_state: State::from_file(&format!("test-data/{}-solo.json", S::NAME))?, - para_state: State::from_file(&format!("test-data/{}-para.json", S::NAME))?, - shell_state: State { - map: mem::take(&mut shell_chain_spec.genesis.raw.top), - _runtime: Default::default(), - }, - shell_chain_spec, - }) - } - - fn process(mut self) -> Result<()> { - self.solo_state.get_value(b"System", b"Number", "", &mut *NOW.write().unwrap()); - - let _guard = NOW.read().unwrap(); - - assert!(*_guard != 0); - - self.process_system() - .process_indices() - .process_vesting() - .process_proxy() - .process_staking() - .process_evm(); - - self.save() - } - - fn save(mut self) -> Result<()> { - log::info!("saving processed chain spec"); - - mem::swap(&mut self.shell_state.map, &mut self.shell_chain_spec.genesis.raw.top); - - let mut f = File::create(format!("test-data/{}-processed.json", S::NAME))?; - let v = serde_json::to_vec(&self.shell_chain_spec)?; - - f.write_all(&v)?; - - Ok(()) - } -} - -pub struct State { - map: Map, - _runtime: PhantomData, -} -impl State { - fn from_file(path: &str) -> Result { - Ok(Self { - map: from_file::(path)?.genesis.raw.top, - _runtime: >::default(), - }) - } - - fn insert_raw_key_raw_value(&mut self, key: String, value: String) -> &mut Self { - self.map.insert(key, value); - - self - } - - fn insert_raw_key_value(&mut self, key: String, value: E) -> &mut Self - where - E: Encode, - { - self.map.insert(key, encode_value(value)); - - self - } - - fn take_raw_map( - &mut self, - prefix: &str, - buffer: &mut Map, - process_key: F, - ) -> &mut Self - where - F: Fn(&str, &str) -> String, - { - self.map.retain(|k, v| { - if k.starts_with(prefix) { - buffer.insert(process_key(k, prefix), v.to_owned()); - - false - } else { - true - } - }); - - self - } - - fn insert_raw_key_map(&mut self, pairs: Map) -> &mut Self { - pairs.into_iter().for_each(|(k, v)| { - if self.map.contains_key(&k) { - log::error!("key({k}) has already existed, overriding"); - } - - self.map.insert(k, v); - }); - - self - } - - fn get_value(&self, pallet: &[u8], item: &[u8], hash: &str, value: &mut D) -> &Self - where - D: Decode, - { - let key = full_key(pallet, item, hash); - - if let Some(v) = self.map.get(&key) { - match decode(v) { - Ok(v) => *value = v, - Err(e) => log::error!( - "failed to decode `{}::{}::{hash}({v})`, due to `{e}`", - String::from_utf8_lossy(pallet), - String::from_utf8_lossy(item), - ), - } - } else { - log::error!( - "key not found `{}::{}::{hash}`", - String::from_utf8_lossy(pallet), - String::from_utf8_lossy(item), - ); - } - - self - } - - fn take_value(&mut self, pallet: &[u8], item: &[u8], hash: &str, value: &mut D) -> &mut Self - where - D: Decode, - { - let key = full_key(pallet, item, hash); - - if let Some(v) = self.map.remove(&key) { - match decode(&v) { - Ok(v) => *value = v, - Err(e) => log::error!( - "failed to decode `{}::{}::{hash}({v})`, due to `{e}`", - String::from_utf8_lossy(pallet), - String::from_utf8_lossy(item) - ), - } - } else { - log::error!( - "key not found `{}::{}::{hash}`", - String::from_utf8_lossy(pallet), - String::from_utf8_lossy(item), - ); - } - - self - } - - fn insert_value(&mut self, pallet: &[u8], item: &[u8], hash: &str, value: E) -> &mut Self - where - E: Encode, - { - self.map.insert(full_key(pallet, item, hash), encode_value(value)); - - self - } - - fn mutate_value(&mut self, pallet: &[u8], item: &[u8], hash: &str, f: F) -> &mut Self - where - D: Default + Encode + Decode, - F: FnOnce(&mut D), - { - let mut v = D::default(); - - self.get_value(pallet, item, hash, &mut v); - - f(&mut v); - - self.insert_value(pallet, item, hash, v); - - self - } - - fn take_map( - &mut self, - pallet: &[u8], - item: &[u8], - buffer: &mut Map, - process_key: F, - ) -> &mut Self - where - D: Decode, - F: Fn(&str, &str) -> String, - { - let len = buffer.len(); - let prefix = item_key(pallet, item); - - self.map.retain(|full_key, v| { - if full_key.starts_with(&prefix) { - match decode(v) { - Ok(v) => { - buffer.insert(process_key(full_key, &prefix), v); - }, - Err(e) => log::error!("failed to decode `{full_key}:{v}`, due to `{e}`"), - } - - false - } else { - true - } - }); - - if buffer.len() == len { - log::info!( - "no new item inserted for {}::{}", - String::from_utf8_lossy(pallet), - String::from_utf8_lossy(item) - ); - } - - self - } - - fn insert_map(&mut self, pairs: Map, process_key: F) -> &mut Self - where - E: Encode, - F: Fn(&str) -> String, - { - pairs.into_iter().for_each(|(k, v)| { - self.map.insert(process_key(&k), encode_value(v)); - }); - - self - } - - fn contains_key(&self, key: &str) -> bool { - self.map.contains_key(key) - } - - // fn inc_consumers(&mut self, who: &str) {} - - // fn transfer(&mut self, from: &str, to: &str, amount: u128) {} - - fn unreserve(&mut self, account_id_32: A, amount: u128) - where - A: AsRef<[u8]>, - { - let account_id_32 = account_id_32.as_ref(); - let (p, i, h) = if is_evm_address(account_id_32) { - (&b"System"[..], &b"Account"[..], &account_id_32[11..31]) - } else { - (&b"AccountMigration"[..], &b"Accounts"[..], account_id_32) - }; - - self.mutate_value(p, i, &blake2_128_concat_to_string(h), |a: &mut AccountInfo| { - a.data.free += amount; - a.data.reserved -= amount; - }); - } -} - -fn from_file(path: &str) -> Result -where - D: DeserializeOwned, -{ - log::info!("load data from {path:?}"); - - let mut f = File::open(path)?; - let mut v = Vec::new(); - - f.read_to_end(&mut v)?; - - Ok(serde_json::from_slice(&v)?) -} - -fn item_key(pallet: &[u8], item: &[u8]) -> String { - let k = substorager::storage_key(pallet, item); - - array_bytes::bytes2hex("0x", &k.0) -} - -fn full_key(pallet: &[u8], item: &[u8], hash: &str) -> String { - format!("{}{hash}", item_key(pallet, item)) -} - -fn encode_value(v: V) -> String -where - V: Encode, -{ - array_bytes::bytes2hex("0x", &v.encode()) -} - -fn decode(hex: &str) -> Result -where - D: Decode, -{ - let v = array_bytes::hex2bytes(hex).map_err(|e| anyhow::anyhow!("{e:?}"))?; - - Ok(D::decode(&mut &*v)?) -} - -// twox128(pallet) + twox128(item) -> twox128(pallet) + twox128(item) -fn get_identity_key(key: &str, _: &str) -> String { - key.into() -} - -// twox128(pallet) + twox128(item) + *(item_key) -> *(item_key) -fn get_hashed_key(full_key: &str, item_key: &str) -> String { - full_key.trim_start_matches(item_key).into() -} - -// twox128(pallet) + twox128(item) + *_concat(account_id_32) -> account_id_32 -fn get_last_64(key: &str) -> String { - format!("0x{}", &key[key.len() - 64..]) -} - -fn replace_first_match(key: &str, from: &str, to: &str) -> String { - key.replacen(from, to, 1) -} - -fn blake2_128_concat_to_string(data: D) -> String -where - D: AsRef<[u8]>, -{ - array_bytes::bytes2hex("", subhasher::blake2_128_concat(data)) -} - -fn is_evm_address(address: &[u8]) -> bool { - address.starts_with(b"dvm:") - && address[1..31].iter().fold(address[0], |checksum, &b| checksum ^ b) == address[31] -} diff --git a/tool/state-processor/src/processor.rs b/tool/state-processor/src/processor.rs new file mode 100644 index 00000000..4fb1e397 --- /dev/null +++ b/tool/state-processor/src/processor.rs @@ -0,0 +1,316 @@ +// std +use std::{ + fs::File, + io::{Read, Write}, + marker::PhantomData, + mem, + sync::RwLock, +}; +// darwinia +use crate::*; +// crates.io +use anyhow::Result; +use fxhash::FxHashMap; +use once_cell::sync::Lazy; +use parity_scale_codec::{Decode, Encode}; +use serde::de::DeserializeOwned; +// hack-ink +use subspector::ChainSpec; + +pub type Map = FxHashMap; + +pub static NOW: Lazy> = Lazy::new(|| RwLock::new(0)); + +pub struct Processor { + pub solo_state: State, + pub para_state: State<()>, + pub shell_state: State<()>, + pub shell_chain_spec: ChainSpec, +} +impl Processor +where + S: Configurable, +{ + pub fn new() -> Result { + build_spec(S::NAME)?; + download_specs(S::NAME)?; + + let mut shell_chain_spec = from_file::(&format!("data/{}-shell.json", S::NAME))?; + + Ok(Self { + solo_state: State::from_file(&format!("data/{}-solo.json", S::NAME))?, + para_state: State::from_file(&format!("data/{}-para.json", S::NAME))?, + shell_state: State { + map: mem::take(&mut shell_chain_spec.genesis.raw.top), + _runtime: Default::default(), + }, + shell_chain_spec, + }) + } + + pub fn process(mut self) -> Result<()> { + self.solo_state.get_value(b"System", b"Number", "", &mut *NOW.write().unwrap()); + + let _guard = NOW.read().unwrap(); + + assert!(*_guard != 0); + + self.process_system() + .process_indices() + .process_vesting() + .process_proxy() + .process_staking() + .process_evm(); + + self.save() + } + + pub fn save(mut self) -> Result<()> { + log::info!("saving processed chain spec"); + + mem::swap(&mut self.shell_state.map, &mut self.shell_chain_spec.genesis.raw.top); + + let mut f = File::create(format!("data/{}-processed.json", S::NAME))?; + let v = serde_json::to_vec(&self.shell_chain_spec)?; + + f.write_all(&v)?; + + Ok(()) + } +} + +pub struct State { + map: Map, + _runtime: PhantomData, +} +impl State { + pub fn from_file(path: &str) -> Result { + Ok(Self { + map: from_file::(path)?.genesis.raw.top, + _runtime: >::default(), + }) + } + + pub fn insert_raw_key_raw_value(&mut self, key: String, value: String) -> &mut Self { + self.map.insert(key, value); + + self + } + + pub fn insert_raw_key_value(&mut self, key: String, value: E) -> &mut Self + where + E: Encode, + { + self.map.insert(key, encode_value(value)); + + self + } + + pub fn take_raw_map( + &mut self, + prefix: &str, + buffer: &mut Map, + process_key: F, + ) -> &mut Self + where + F: Fn(&str, &str) -> String, + { + self.map.retain(|k, v| { + if k.starts_with(prefix) { + buffer.insert(process_key(k, prefix), v.to_owned()); + + false + } else { + true + } + }); + + self + } + + pub fn insert_raw_key_map(&mut self, pairs: Map) -> &mut Self { + pairs.into_iter().for_each(|(k, v)| { + if self.map.contains_key(&k) { + log::error!("key({k}) has already existed, overriding"); + } + + self.map.insert(k, v); + }); + + self + } + + pub fn get_value(&self, pallet: &[u8], item: &[u8], hash: &str, value: &mut D) -> &Self + where + D: Decode, + { + let key = full_key(pallet, item, hash); + + if let Some(v) = self.map.get(&key) { + match decode(v) { + Ok(v) => *value = v, + Err(e) => log::error!( + "failed to decode `{}::{}::{hash}({v})`, due to `{e}`", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item), + ), + } + } else { + log::error!( + "key not found `{}::{}::{hash}`", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item), + ); + } + + self + } + + pub fn take_value( + &mut self, + pallet: &[u8], + item: &[u8], + hash: &str, + value: &mut D, + ) -> &mut Self + where + D: Decode, + { + let key = full_key(pallet, item, hash); + + if let Some(v) = self.map.remove(&key) { + match decode(&v) { + Ok(v) => *value = v, + Err(e) => log::error!( + "failed to decode `{}::{}::{hash}({v})`, due to `{e}`", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item) + ), + } + } else { + log::error!( + "key not found `{}::{}::{hash}`", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item), + ); + } + + self + } + + pub fn insert_value(&mut self, pallet: &[u8], item: &[u8], hash: &str, value: E) -> &mut Self + where + E: Encode, + { + self.map.insert(full_key(pallet, item, hash), encode_value(value)); + + self + } + + pub fn mutate_value(&mut self, pallet: &[u8], item: &[u8], hash: &str, f: F) -> &mut Self + where + D: Default + Encode + Decode, + F: FnOnce(&mut D), + { + let mut v = D::default(); + + self.get_value(pallet, item, hash, &mut v); + + f(&mut v); + + self.insert_value(pallet, item, hash, v); + + self + } + + pub fn take_map( + &mut self, + pallet: &[u8], + item: &[u8], + buffer: &mut Map, + process_key: F, + ) -> &mut Self + where + D: Decode, + F: Fn(&str, &str) -> String, + { + let len = buffer.len(); + let prefix = item_key(pallet, item); + + self.map.retain(|full_key, v| { + if full_key.starts_with(&prefix) { + match decode(v) { + Ok(v) => { + buffer.insert(process_key(full_key, &prefix), v); + }, + Err(e) => log::error!("failed to decode `{full_key}:{v}`, due to `{e}`"), + } + + false + } else { + true + } + }); + + if buffer.len() == len { + log::info!( + "no new item inserted for {}::{}", + String::from_utf8_lossy(pallet), + String::from_utf8_lossy(item) + ); + } + + self + } + + pub fn insert_map(&mut self, pairs: Map, process_key: F) -> &mut Self + where + E: Encode, + F: Fn(&str) -> String, + { + pairs.into_iter().for_each(|(k, v)| { + self.map.insert(process_key(&k), encode_value(v)); + }); + + self + } + + pub fn contains_key(&self, key: &str) -> bool { + self.map.contains_key(key) + } + + // pub fn inc_consumers(&mut self, who: &str) {} + + // pub fn transfer(&mut self, from: &str, to: &str, amount: u128) {} + + pub fn unreserve(&mut self, account_id_32: A, amount: u128) + where + A: AsRef<[u8]>, + { + let account_id_32 = account_id_32.as_ref(); + let (p, i, h) = if is_evm_address(account_id_32) { + (&b"System"[..], &b"Account"[..], &account_id_32[11..31]) + } else { + (&b"AccountMigration"[..], &b"Accounts"[..], account_id_32) + }; + + self.mutate_value(p, i, &blake2_128_concat_to_string(h), |a: &mut AccountInfo| { + a.data.free += amount; + a.data.reserved -= amount; + }); + } +} + +pub fn from_file(path: &str) -> Result +where + D: DeserializeOwned, +{ + log::info!("load data from {path:?}"); + + let mut f = File::open(path)?; + let mut v = Vec::new(); + + f.read_to_end(&mut v)?; + + Ok(serde_json::from_slice(&v)?) +} diff --git a/tool/state-processor/src/staking/README.md b/tool/state-processor/src/staking/README.md index deffe19b..4726c829 100644 --- a/tool/state-processor/src/staking/README.md +++ b/tool/state-processor/src/staking/README.md @@ -1,4 +1,4 @@ ### Process steps -- take solo `Staking::Bonded`, `Staking::Ledger`, `Staking::RingPool`, `Staking::KtonPool` and `Staking::LivingTime` +- take solo `Staking::Ledger`, `Staking::RingPool`, `Staking::KtonPool` and `Staking::LivingTime` - adjust decimals and block number, convert ledger, adjust unstaking duration then set `AccountMigration::Ledgers` and `AccountMigration::Deposits` - set `Staking::RingPool` and `Staking::KtonPool` diff --git a/tool/state-processor/src/staking/mod.rs b/tool/state-processor/src/staking/mod.rs index f4060b4b..908ae41d 100644 --- a/tool/state-processor/src/staking/mod.rs +++ b/tool/state-processor/src/staking/mod.rs @@ -5,7 +5,6 @@ impl Processor { pub fn process_staking(&mut self) -> &mut Self { // Storage items. // https://github.dev/darwinia-network/darwinia-common/blob/darwinia-v0.12.5/frame/staking/src/lib.rs#L560 - let mut bonded = >::default(); let mut ledgers = >::default(); let mut ring_pool_storage = u128::default(); let mut kton_pool_storage = u128::default(); @@ -13,19 +12,13 @@ impl Processor { let mut kton_pool = u128::default(); let mut elapsed_time = u64::default(); - log::info!("take solo `Staking::Bonded`, `Staking::Ledger`, `Staking::RingPool`, `Staking::KtonPool` and `Staking::LivingTime`"); + log::info!("take solo `Staking::Ledger`, `Staking::RingPool`, `Staking::KtonPool` and `Staking::LivingTime`"); self.solo_state - .take_raw_map(&item_key(b"Staking", b"Bonded"), &mut bonded, |key, from| { - replace_first_match(key, from, &item_key(b"AccountMigration", b"Bonded")) - }) .take_map(b"Staking", b"Ledger", &mut ledgers, get_identity_key) .take_value(b"Staking", b"RingPool", "", &mut ring_pool_storage) .take_value(b"Staking", b"KtonPool", "", &mut kton_pool_storage) .take_value(b"Staking", b"LivingTime", "", &mut elapsed_time); - log::info!("set `Staking::Bonded`"); - self.shell_state.insert_raw_key_map(bonded); - log::info!("adjust decimals and block number, convert ledger, adjust unstaking duration then set `AccountMigration::Ledgers` and `AccountMigration::Deposits`"); { let staking_ik = item_key(b"AccountMigration", b"Ledgers"); diff --git a/tool/state-processor/src/system/mod.rs b/tool/state-processor/src/system/mod.rs index 4b5309d8..ae5ce210 100644 --- a/tool/state-processor/src/system/mod.rs +++ b/tool/state-processor/src/system/mod.rs @@ -1,3 +1,5 @@ +// crates.io +use parity_scale_codec::Encode; // darwinia use crate::*; diff --git a/tool/state-processor/src/util.rs b/tool/state-processor/src/util.rs new file mode 100644 index 00000000..ec3365fe --- /dev/null +++ b/tool/state-processor/src/util.rs @@ -0,0 +1,95 @@ +// std +use std::{ + fs::File, + process::{Command, Stdio}, +}; +// crates.io +use parity_scale_codec::{Decode, Encode}; +use tar::Archive; +use zstd::Decoder; +// darwinia +use crate::*; + +pub fn item_key(pallet: &[u8], item: &[u8]) -> String { + let k = substorager::storage_key(pallet, item); + + array_bytes::bytes2hex("0x", &k.0) +} + +pub fn full_key(pallet: &[u8], item: &[u8], hash: &str) -> String { + format!("{}{hash}", item_key(pallet, item)) +} + +pub fn encode_value(v: V) -> String +where + V: Encode, +{ + array_bytes::bytes2hex("0x", &v.encode()) +} + +pub fn decode(hex: &str) -> Result +where + D: Decode, +{ + let v = array_bytes::hex2bytes(hex).map_err(|e| anyhow::anyhow!("{e:?}"))?; + + Ok(D::decode(&mut &*v)?) +} + +// twox128(pallet) + twox128(item) -> twox128(pallet) + twox128(item) +pub fn get_identity_key(key: &str, _: &str) -> String { + key.into() +} + +// twox128(pallet) + twox128(item) + *(item_key) -> *(item_key) +pub fn get_hashed_key(full_key: &str, item_key: &str) -> String { + full_key.trim_start_matches(item_key).into() +} + +// twox128(pallet) + twox128(item) + *_concat(account_id_32) -> account_id_32 +pub fn get_last_64(key: &str) -> String { + format!("0x{}", &key[key.len() - 64..]) +} + +pub fn replace_first_match(key: &str, from: &str, to: &str) -> String { + key.replacen(from, to, 1) +} + +pub fn blake2_128_concat_to_string(data: D) -> String +where + D: AsRef<[u8]>, +{ + array_bytes::bytes2hex("", subhasher::blake2_128_concat(data)) +} + +pub fn is_evm_address(address: &[u8]) -> bool { + address.starts_with(b"dvm:") + && address[1..31].iter().fold(address[0], |checksum, &b| checksum ^ b) == address[31] +} + +pub fn build_spec(chain: &str) -> Result<()> { + Command::new("../../target/release/darwinia") + .args(["build-spec", "--chain", &format!("{chain}-genesis")]) + .stdout(Stdio::from(File::create(format!("data/{chain}-shell.json"))?)) + .output()?; + + Ok(()) +} + +pub fn download_specs(chain: &str) -> Result<()> { + let decoder = Decoder::new( + ureq::get(&format!( + "https://github.com/darwinia-network/darwinia-2.0/releases/download/{chain}2/{chain}-state.tar.zst" + )) + .call()? + .into_reader(), + )?; + + for e in Archive::new(decoder).entries()? { + let mut e = e?; + + e.unpack(format!("data/{}", e.path()?.to_string_lossy()))?; + } + + Ok(()) +}