From 22b86994e74f66ba4df864874d6474990ba3a7b6 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 27 Jan 2021 17:23:34 +0100 Subject: [PATCH 01/10] validation extension in sp_io --- primitives/io/Cargo.toml | 18 ++++++++-------- primitives/io/src/lib.rs | 44 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 01ea58e87e3e2..42355e6c58a9f 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -17,16 +17,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "1.3.6", default-features = false } hash-db = { version = "0.15.2", default-features = false } -sp-core = { version = "2.0.0", default-features = false, path = "../core" } -sp-keystore = { version = "0.8.0", default-features = false, optional = true, path = "../keystore" } -sp-std = { version = "2.0.0", default-features = false, path = "../std" } +sp-core = { version = "2.0.0", default-features = false } +sp-keystore = { version = "0.8.0", default-features = false, optional = true } +sp-std = { version = "2.0.0", default-features = false } libsecp256k1 = { version = "0.3.4", optional = true } -sp-state-machine = { version = "0.8.0", optional = true, path = "../state-machine" } -sp-wasm-interface = { version = "2.0.0", path = "../wasm-interface", default-features = false } -sp-runtime-interface = { version = "2.0.0", default-features = false, path = "../runtime-interface" } -sp-trie = { version = "2.0.0", optional = true, path = "../trie" } -sp-externalities = { version = "0.8.0", optional = true, path = "../externalities" } -sp-tracing = { version = "2.0.0", default-features = false, path = "../tracing" } +sp-state-machine = { version = "0.8.0", optional = true } +sp-wasm-interface = { version = "2.0.0", default-features = false } +sp-runtime-interface = { version = "2.0.0", default-features = false } +sp-trie = { version = "2.0.0", optional = true } +sp-externalities = { version = "0.8.0", optional = true } +sp-tracing = { version = "2.0.0", default-features = false } log = { version = "0.4.8", optional = true } futures = { version = "0.3.1", features = ["thread-pool"], optional = true } parking_lot = { version = "0.11.1", optional = true } diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 397dd3c21712a..5b9ac70feb3e5 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1292,7 +1292,49 @@ pub trait RuntimeTasks { runtime_spawn.join(handle) }).expect("`RuntimeTasks::join`: called outside of externalities context") } - } +} + +use sp_std::boxed::Box; + + +#[runtime_interface] +pub trait Validation { + /// TODO + fn validation_code(&mut self) -> Vec { + use sp_externalities::ExternalitiesExt; + let extension = self.extension::() + .expect("Validation extension missing, this should only be call in a validation context."); + extension.validation_code() + } +} + +#[cfg(feature = "std")] +sp_externalities::decl_extension! { + /// executor extension. + pub struct ValidationExt(Box); +} + +#[cfg(feature = "std")] +impl ValidationExt { + /// New instance of task executor extension. + pub fn new(validation_ext: impl Validation) -> Self { + Self(Box::new(validation_ext)) + } +} + +/// Base methods to implement validation extension. +pub trait Validation: Send + 'static { + /// Get the validation code currently running. + /// This can be use to check validity or to complete + /// proofs. + fn validation_code(&self) -> Vec; +} + +impl Validation for &'static [u8] { + fn validation_code(&self) -> Vec { + self.to_vec() + } +} /// Allocator used by Substrate when executing the Wasm runtime. #[cfg(not(feature = "std"))] From 4de9f7a30f60fe56e247cf32332016c7d2620baa Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 27 Jan 2021 17:36:40 +0100 Subject: [PATCH 02/10] need paths --- primitives/io/Cargo.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/primitives/io/Cargo.toml b/primitives/io/Cargo.toml index 42355e6c58a9f..01ea58e87e3e2 100644 --- a/primitives/io/Cargo.toml +++ b/primitives/io/Cargo.toml @@ -17,16 +17,16 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "1.3.6", default-features = false } hash-db = { version = "0.15.2", default-features = false } -sp-core = { version = "2.0.0", default-features = false } -sp-keystore = { version = "0.8.0", default-features = false, optional = true } -sp-std = { version = "2.0.0", default-features = false } +sp-core = { version = "2.0.0", default-features = false, path = "../core" } +sp-keystore = { version = "0.8.0", default-features = false, optional = true, path = "../keystore" } +sp-std = { version = "2.0.0", default-features = false, path = "../std" } libsecp256k1 = { version = "0.3.4", optional = true } -sp-state-machine = { version = "0.8.0", optional = true } -sp-wasm-interface = { version = "2.0.0", default-features = false } -sp-runtime-interface = { version = "2.0.0", default-features = false } -sp-trie = { version = "2.0.0", optional = true } -sp-externalities = { version = "0.8.0", optional = true } -sp-tracing = { version = "2.0.0", default-features = false } +sp-state-machine = { version = "0.8.0", optional = true, path = "../state-machine" } +sp-wasm-interface = { version = "2.0.0", path = "../wasm-interface", default-features = false } +sp-runtime-interface = { version = "2.0.0", default-features = false, path = "../runtime-interface" } +sp-trie = { version = "2.0.0", optional = true, path = "../trie" } +sp-externalities = { version = "0.8.0", optional = true, path = "../externalities" } +sp-tracing = { version = "2.0.0", default-features = false, path = "../tracing" } log = { version = "0.4.8", optional = true } futures = { version = "0.3.1", features = ["thread-pool"], optional = true } parking_lot = { version = "0.11.1", optional = true } From f0ea2af03f1bd078a0b55c191eea2488da638363 Mon Sep 17 00:00:00 2001 From: cheme Date: Wed, 27 Jan 2021 17:43:38 +0100 Subject: [PATCH 03/10] arc impl --- primitives/io/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 5b9ac70feb3e5..42d1409ec4f14 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1336,6 +1336,13 @@ impl Validation for &'static [u8] { } } +#[cfg(feature = "std")] +impl Validation for std::sync::Arc { + fn validation_code(&self) -> Vec { + self.deref().validation_code() + } +} + /// Allocator used by Substrate when executing the Wasm runtime. #[cfg(not(feature = "std"))] struct WasmAllocator; From 71202a940060d875543d24a430ec60cae2ca1e9c Mon Sep 17 00:00:00 2001 From: cheme Date: Thu, 28 Jan 2021 10:09:56 +0100 Subject: [PATCH 04/10] missing host function in executor --- primitives/io/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index 42d1409ec4f14..ee7550aae1a7e 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1411,6 +1411,7 @@ pub type SubstrateHostFunctions = ( crate::trie::HostFunctions, offchain_index::HostFunctions, runtime_tasks::HostFunctions, + validation::HostFunctions, ); #[cfg(test)] From f848b4434fa2925f7b7f4b31b3d2023e7e58a7f8 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Feb 2021 10:17:33 +0100 Subject: [PATCH 05/10] io to pkdot --- primitives/io/src/lib.rs | 52 +--------------------------------------- 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/primitives/io/src/lib.rs b/primitives/io/src/lib.rs index ee7550aae1a7e..397dd3c21712a 100644 --- a/primitives/io/src/lib.rs +++ b/primitives/io/src/lib.rs @@ -1292,56 +1292,7 @@ pub trait RuntimeTasks { runtime_spawn.join(handle) }).expect("`RuntimeTasks::join`: called outside of externalities context") } -} - -use sp_std::boxed::Box; - - -#[runtime_interface] -pub trait Validation { - /// TODO - fn validation_code(&mut self) -> Vec { - use sp_externalities::ExternalitiesExt; - let extension = self.extension::() - .expect("Validation extension missing, this should only be call in a validation context."); - extension.validation_code() - } -} - -#[cfg(feature = "std")] -sp_externalities::decl_extension! { - /// executor extension. - pub struct ValidationExt(Box); -} - -#[cfg(feature = "std")] -impl ValidationExt { - /// New instance of task executor extension. - pub fn new(validation_ext: impl Validation) -> Self { - Self(Box::new(validation_ext)) - } -} - -/// Base methods to implement validation extension. -pub trait Validation: Send + 'static { - /// Get the validation code currently running. - /// This can be use to check validity or to complete - /// proofs. - fn validation_code(&self) -> Vec; -} - -impl Validation for &'static [u8] { - fn validation_code(&self) -> Vec { - self.to_vec() - } -} - -#[cfg(feature = "std")] -impl Validation for std::sync::Arc { - fn validation_code(&self) -> Vec { - self.deref().validation_code() - } -} + } /// Allocator used by Substrate when executing the Wasm runtime. #[cfg(not(feature = "std"))] @@ -1411,7 +1362,6 @@ pub type SubstrateHostFunctions = ( crate::trie::HostFunctions, offchain_index::HostFunctions, runtime_tasks::HostFunctions, - validation::HostFunctions, ); #[cfg(test)] From 1979b6a2e2f34be00f4f27572453beea811e4a70 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Feb 2021 13:03:31 +0100 Subject: [PATCH 06/10] decode function. --- Cargo.lock | 8 +- Cargo.toml | 4 + primitives/trie/Cargo.toml | 2 +- primitives/trie/src/lib.rs | 5 ++ primitives/trie/src/trie_codec.rs | 140 ++++++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 6 deletions(-) create mode 100644 primitives/trie/src/trie_codec.rs diff --git a/Cargo.lock b/Cargo.lock index 7421af5ff1b5e..06e430bb97469 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2143,8 +2143,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d23bd4e7b5eda0d0f3a307e8b381fdc8ba9000f26fbe912250c0a4cc3956364a" +source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#ccb001e53234adb85b0c00b031bbc56151018d43" [[package]] name = "hash256-std-hasher" @@ -9755,9 +9754,8 @@ dependencies = [ [[package]] name = "trie-db" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc176c377eb24d652c9c69c832c832019011b6106182bf84276c66b66d5c9a6" +version = "0.22.3" +source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#ccb001e53234adb85b0c00b031bbc56151018d43" dependencies = [ "hash-db", "hashbrown", diff --git a/Cargo.toml b/Cargo.toml index 12e79490ef6b0..59f5f09e1e975 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -255,3 +255,7 @@ zeroize = { opt-level = 3 } [profile.release] # Substrate runtime requires unwinding. panic = "unwind" + +[patch.crates-io] +trie-db = { git = 'https://github.com/cheme/trie', branch = 'skip_compact_values_alt' } +hash-db = { git = 'https://github.com/cheme/trie', branch = 'skip_compact_values_alt' } diff --git a/primitives/trie/Cargo.toml b/primitives/trie/Cargo.toml index 4392f01d222aa..bf52f08c3fc12 100644 --- a/primitives/trie/Cargo.toml +++ b/primitives/trie/Cargo.toml @@ -21,7 +21,7 @@ harness = false codec = { package = "parity-scale-codec", version = "1.3.6", default-features = false } sp-std = { version = "2.0.0", default-features = false, path = "../std" } hash-db = { version = "0.15.2", default-features = false } -trie-db = { version = "0.22.2", default-features = false } +trie-db = { version = "0.22.3", default-features = false } trie-root = { version = "0.16.0", default-features = false } memory-db = { version = "0.25.0", default-features = false } sp-core = { version = "2.0.0", default-features = false, path = "../core" } diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 572283f1c027e..3ecad1d5b0318 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -23,6 +23,7 @@ mod error; mod node_header; mod node_codec; mod storage_proof; +mod trie_codec; mod trie_stream; use sp_std::{boxed::Box, marker::PhantomData, vec::Vec, borrow::Borrow}; @@ -39,12 +40,16 @@ pub use storage_proof::StorageProof; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, + LazyFetcher, }; /// Various re-exports from the `memory-db` crate. pub use memory_db::KeyFunction; pub use memory_db::prefixed_key; /// Various re-exports from the `hash-db` crate. pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; +/// Trie codec reexport, mainly child trie support +/// for trie compact proof. +pub use trie_codec::{decode_compact}; #[derive(Default)] /// substrate trie layout diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs new file mode 100644 index 0000000000000..711b59c7dcadd --- /dev/null +++ b/primitives/trie/src/trie_codec.rs @@ -0,0 +1,140 @@ +// This file is part of Substrate. + +// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Compact proof support. +//! +//! This uses compact proof from trie crate and extends +//! it to substrate specific layout and child trie system. + +use crate::{EMPTY_PREFIX, HashDBT, LazyFetcher, + TrieHash, TrieError, TrieConfiguration}; + +type VerifyError = crate::VerifyError, Box>>; + +fn verify_error(error: Box>) -> VerifyError { + VerifyError::::DecodeError(error) +} + +/// Decode a compact proof. +/// +/// Takes as input a destination `db` for decoded node and `encoded` +/// an iterator of compact encoded nodes. +/// +/// Also allows optionally injecting specified value in +/// top trie proof with `know_keys` and the lazy +/// associated `fetcher`. +/// +/// Child trie are decoded in order of child trie root present +/// in the top trie. +pub fn decode_compact<'a, L, DB, I, F, K>( + db: &mut DB, + encoded: I, + fetcher: F, + known_keys: Option, + expected_root: Option<&TrieHash>, +) -> Result, VerifyError> + where + L: TrieConfiguration, + DB: HashDBT + hash_db::HashDBRef, + I: IntoIterator, + F: LazyFetcher<'a>, + K: IntoIterator, +{ + let mut nodes_iter = encoded.into_iter(); + let (top_root, _nb_used) = if let Some(known_keys) = known_keys { + trie_db::decode_compact_with_known_values::( + db, + &mut nodes_iter, + fetcher, + known_keys, + false, // current use of compact do to escape empty value. + ).map_err(verify_error::)? + } else { + trie_db::decode_compact_from_iter::( + db, + &mut nodes_iter, + ).map_err(verify_error::)? + }; + + if let Some(expected_root) = expected_root { + if expected_root != &top_root { + return Err(VerifyError::::RootMismatch(expected_root.clone())); + } + } + + let mut child_tries = Vec::new(); + { + // fetch child trie roots + let trie = crate::TrieDB::::new(db, &top_root).map_err(verify_error::)?; + + use trie_db::Trie; + let mut iter = trie.iter().map_err(verify_error::)?; + + let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX; + if iter.seek(childtrie_roots).is_ok() { + loop { + match iter.next() { + Some(Ok((key, value))) if key.starts_with(childtrie_roots) => { + // we expect all default child trie root to be correctly encoded. + // see other child trie functions. + let mut root = TrieHash::::default(); + // still in a proof so prevent panic + if root.as_mut().len() != value.as_slice().len() { + return Err(VerifyError::::RootMismatch(Default::default())); + } + root.as_mut().copy_from_slice(value.as_ref()); + child_tries.push(root); + }, + // allow incomplete database error: we only + // require access to data in the proof. + Some(Err(error)) => match *error { + trie_db::TrieError::IncompleteDatabase(..) => (), + e => panic!("{:?}", e), + }, + _ => break, + } + } + } + } + + if !HashDBT::::contains(db, &top_root, EMPTY_PREFIX) { + return Err(VerifyError::::IncompleteProof); + } + + for child_root in child_tries.into_iter() { + if !HashDBT::::contains(db, &child_root, EMPTY_PREFIX) { + // child proof are allowed to be missing (unused root can be included + // due to trie structure modification). + continue; + } + + let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( + db, + &mut nodes_iter, + ).map_err(verify_error::)?; + + if child_root != top_root { + return Err(VerifyError::::RootMismatch(child_root)); + } + } + + if nodes_iter.next().is_some() { + return Err(VerifyError::::ExtraneousNode); + } + + Ok(top_root) +} From 927a096737416c5e83587eb0b7b9b06bae2da83d Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Feb 2021 14:53:06 +0100 Subject: [PATCH 07/10] encode primitive. --- primitives/trie/src/lib.rs | 4 +- primitives/trie/src/storage_proof.rs | 13 ++++++ primitives/trie/src/trie_codec.rs | 69 +++++++++++++++++++++++++++- 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/primitives/trie/src/lib.rs b/primitives/trie/src/lib.rs index 3ecad1d5b0318..5a4cff6c9d5be 100644 --- a/primitives/trie/src/lib.rs +++ b/primitives/trie/src/lib.rs @@ -36,7 +36,7 @@ pub use error::Error; pub use trie_stream::TrieStream; /// The Substrate format implementation of `NodeCodec`. pub use node_codec::NodeCodec; -pub use storage_proof::StorageProof; +pub use storage_proof::{StorageProof, CompactProof}; /// Various re-exports from the `trie-db` crate. pub use trie_db::{ Trie, TrieMut, DBValue, Recorder, CError, Query, TrieLayout, TrieConfiguration, nibble_ops, TrieDBIterator, @@ -49,7 +49,7 @@ pub use memory_db::prefixed_key; pub use hash_db::{HashDB as HashDBT, EMPTY_PREFIX}; /// Trie codec reexport, mainly child trie support /// for trie compact proof. -pub use trie_codec::{decode_compact}; +pub use trie_codec::{decode_compact, encode_compact}; #[derive(Default)] /// substrate trie layout diff --git a/primitives/trie/src/storage_proof.rs b/primitives/trie/src/storage_proof.rs index f0b2bfd4bc3d3..96c2be3460393 100644 --- a/primitives/trie/src/storage_proof.rs +++ b/primitives/trie/src/storage_proof.rs @@ -31,6 +31,12 @@ pub struct StorageProof { trie_nodes: Vec>, } +/// Storage proof in compact form. +#[derive(Debug, PartialEq, Eq, Clone, Encode, Decode)] +pub struct CompactProof { + pub encoded_nodes: Vec>, +} + impl StorageProof { /// Constructs a storage proof from a subset of encoded trie nodes in a storage backend. pub fn new(trie_nodes: Vec>) -> Self { @@ -77,6 +83,13 @@ impl StorageProof { } } +impl CompactProof { + /// Return an iterator on the compact encoded nodes. + pub fn iter_compact_encoded_nodes(&self) -> impl Iterator { + self.encoded_nodes.iter().map(Vec::as_slice) + } +} + /// An iterator over trie nodes constructed from a storage proof. The nodes are not guaranteed to /// be traversed in any particular order. pub struct StorageProofNodeIterator { diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 711b59c7dcadd..7c0e9d3460b4b 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -21,7 +21,9 @@ //! it to substrate specific layout and child trie system. use crate::{EMPTY_PREFIX, HashDBT, LazyFetcher, - TrieHash, TrieError, TrieConfiguration}; + TrieHash, TrieError, TrieConfiguration, CompactProof, StorageProof}; +use sp_std::boxed::Box; +use sp_std::vec::Vec; type VerifyError = crate::VerifyError, Box>>; @@ -103,7 +105,7 @@ pub fn decode_compact<'a, L, DB, I, F, K>( // require access to data in the proof. Some(Err(error)) => match *error { trie_db::TrieError::IncompleteDatabase(..) => (), - e => panic!("{:?}", e), + e => return Err(VerifyError::::DecodeError(Box::new(e))), }, _ => break, } @@ -138,3 +140,66 @@ pub fn decode_compact<'a, L, DB, I, F, K>( Ok(top_root) } + +pub fn encode_compact<'a, L, I>( + proof: StorageProof, + root: TrieHash, + to_skip: I, +) -> Result>> + where + L: TrieConfiguration, + I: IntoIterator + 'a, +{ + let mut child_tries = Vec::new(); + let partial_db = proof.into_memory_db(); + let mut compact_proof = { + let trie = crate::TrieDB::::new(&partial_db, &root)?; + + use trie_db::Trie; + let mut iter = trie.iter()?; + + let childtrie_roots = sp_core::storage::well_known_keys::DEFAULT_CHILD_STORAGE_KEY_PREFIX; + if iter.seek(childtrie_roots).is_ok() { + loop { + match iter.next() { + Some(Ok((key, value))) if key.starts_with(childtrie_roots) => { + let mut root = TrieHash::::default(); + if root.as_mut().len() != value.as_slice().len() { + return Err(Box::new(trie_db::TrieError::InvalidStateRoot(Default::default()))); + } + root.as_mut().copy_from_slice(value.as_ref()); + child_tries.push(root); + }, + // allow incomplete database error: we only + // require access to data in the proof. + Some(Err(error)) => match *error { + trie_db::TrieError::IncompleteDatabase(..) => (), + e => return Err(Box::new(e)), + }, + _ => break, + } + } + } + + trie_db::encode_compact_skip_conditional_with_key::( + &trie, + trie_db::compact_conditions::skip_given_ordered_keys(to_skip), + false, // We do not escape empty value. + )? + }; + + for child_root in child_tries { + if !HashDBT::::contains(&partial_db, &child_root, EMPTY_PREFIX) { + // child proof are allowed to be missing (unused root can be included + // due to trie structure modification). + continue; + } + + let trie = crate::TrieDB::::new(&partial_db, &child_root)?; + let child_proof = trie_db::encode_compact::(&trie)?; + + compact_proof.extend(child_proof); + } + + Ok(CompactProof { encoded_nodes: compact_proof }) +} From b034ec0b7b451be829fadd1939a9f8d5e8178cf9 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Feb 2021 15:34:23 +0100 Subject: [PATCH 08/10] trailing tab --- primitives/trie/src/trie_codec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index 7c0e9d3460b4b..bfeea8e0293d4 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -1,6 +1,6 @@ // This file is part of Substrate. -// Copyright (C) 2015-2021 Parity Technologies (UK) Ltd. +// Copyright (C) 2021-2021 Parity Technologies (UK) Ltd. // SPDX-License-Identifier: Apache-2.0 // Licensed under the Apache License, Version 2.0 (the "License"); @@ -149,7 +149,7 @@ pub fn encode_compact<'a, L, I>( where L: TrieConfiguration, I: IntoIterator + 'a, -{ +{ let mut child_tries = Vec::new(); let partial_db = proof.into_memory_db(); let mut compact_proof = { From 757ff301ee60ce69a17546ac709bca0e2f00e472 Mon Sep 17 00:00:00 2001 From: cheme Date: Mon, 1 Feb 2021 16:46:20 +0100 Subject: [PATCH 09/10] multiple patch --- utils/wasm-builder/src/wasm_project.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/wasm-builder/src/wasm_project.rs b/utils/wasm-builder/src/wasm_project.rs index 73dc2e13af348..ea28b1dfb9f2c 100644 --- a/utils/wasm-builder/src/wasm_project.rs +++ b/utils/wasm-builder/src/wasm_project.rs @@ -224,7 +224,7 @@ fn create_project_cargo_toml( wasm_workspace_toml.insert("profile".into(), profile.into()); // Add patch section from the project root `Cargo.toml` - if let Some(mut patch) = workspace_toml.remove("patch").and_then(|p| p.try_into::().ok()) { + while let Some(mut patch) = workspace_toml.remove("patch").and_then(|p| p.try_into::
().ok()) { // Iterate over all patches and make the patch path absolute from the workspace root path. patch.iter_mut() .filter_map(|p| From 7dba01bb0cc4929a75012af56c0fbd53fb59ffe3 Mon Sep 17 00:00:00 2001 From: cheme Date: Tue, 2 Feb 2021 11:29:51 +0100 Subject: [PATCH 10/10] fix child trie logic --- Cargo.lock | 4 ++-- primitives/state-machine/src/lib.rs | 29 +++++++++++++++++++++++++++-- primitives/trie/src/trie_codec.rs | 27 +++++++++++++++------------ 3 files changed, 44 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 06e430bb97469..cf927a2e0eca7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2143,7 +2143,7 @@ dependencies = [ [[package]] name = "hash-db" version = "0.15.2" -source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#ccb001e53234adb85b0c00b031bbc56151018d43" +source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#8b8c6481c3360678b734b0f08a5360216bb53039" [[package]] name = "hash256-std-hasher" @@ -9755,7 +9755,7 @@ dependencies = [ [[package]] name = "trie-db" version = "0.22.3" -source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#ccb001e53234adb85b0c00b031bbc56151018d43" +source = "git+https://github.com/cheme/trie?branch=skip_compact_values_alt#8b8c6481c3360678b734b0f08a5360216bb53039" dependencies = [ "hash-db", "hashbrown", diff --git a/primitives/state-machine/src/lib.rs b/primitives/state-machine/src/lib.rs index 6d85b56f8aae2..b8b777fd4f2c3 100644 --- a/primitives/state-machine/src/lib.rs +++ b/primitives/state-machine/src/lib.rs @@ -1434,12 +1434,36 @@ mod tests { #[test] fn prove_read_and_proof_check_works() { + fn test_compact(remote_proof: StorageProof, remote_root: &sp_core::H256) -> StorageProof { + type Layout = sp_trie::Layout; + let compact_remote_proof = sp_trie::encode_compact::( + remote_proof, + remote_root.clone(), + std::iter::empty(), + ).unwrap(); + let mut db = sp_trie::MemoryDB::::new(&[]); + sp_trie::decode_compact::>( + &mut db, + compact_remote_proof.iter_compact_encoded_nodes(), + (), + None, + Some(remote_root), + ).unwrap(); + StorageProof::new(db.drain().into_iter().filter_map(|kv| + if (kv.1).1 > 0 { + Some((kv.1).0) + } else { + None + } + ).collect()) + } let child_info = ChildInfo::new_default(b"sub1"); let child_info = &child_info; // fetch read proof from 'remote' full node let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_root = remote_backend.storage_root(std::iter::empty()).0; let remote_proof = prove_read(remote_backend, &[b"value2"]).unwrap(); + let remote_proof = test_compact(remote_proof, &remote_root); // check proof locally let local_result1 = read_proof_check::( remote_root, @@ -1459,12 +1483,13 @@ mod tests { assert_eq!(local_result2, false); // on child trie let remote_backend = trie_backend::tests::test_trie(); - let remote_root = remote_backend.storage_root(::std::iter::empty()).0; + let remote_root = remote_backend.storage_root(std::iter::empty()).0; let remote_proof = prove_child_read( remote_backend, child_info, &[b"value3"], ).unwrap(); + let remote_proof = test_compact(remote_proof, &remote_root); let local_result1 = read_child_proof_check::( remote_root, remote_proof.clone(), diff --git a/primitives/trie/src/trie_codec.rs b/primitives/trie/src/trie_codec.rs index bfeea8e0293d4..43d630c7f0246 100644 --- a/primitives/trie/src/trie_codec.rs +++ b/primitives/trie/src/trie_codec.rs @@ -117,22 +117,25 @@ pub fn decode_compact<'a, L, DB, I, F, K>( return Err(VerifyError::::IncompleteProof); } + let mut previous_extracted_child_trie = None; for child_root in child_tries.into_iter() { - if !HashDBT::::contains(db, &child_root, EMPTY_PREFIX) { - // child proof are allowed to be missing (unused root can be included - // due to trie structure modification). - continue; + if previous_extracted_child_trie == None { + let (top_root, _) = trie_db::decode_compact_from_iter::( + db, + &mut nodes_iter, + ).map_err(verify_error::)?; + previous_extracted_child_trie = Some(top_root); } - - let (top_root, _nb_used) = trie_db::decode_compact_from_iter::( - db, - &mut nodes_iter, - ).map_err(verify_error::)?; - - if child_root != top_root { - return Err(VerifyError::::RootMismatch(child_root)); + + // we allow skipping child root by only + // decoding next on match. + if Some(child_root) == previous_extracted_child_trie { + previous_extracted_child_trie = None; } } + if let Some(child_root) = previous_extracted_child_trie { + return Err(VerifyError::::RootMismatch(child_root)); + } if nodes_iter.next().is_some() { return Err(VerifyError::::ExtraneousNode);