From 50f04733223efeb626990af892f6d4c9ff387337 Mon Sep 17 00:00:00 2001 From: ZhangTao Date: Wed, 29 Sep 2021 14:05:25 +0800 Subject: [PATCH] [StateService] get historical state (#638) * StateService: add historical state * GetState and FindState; Results in pages * use root hash * StateService: add from in mpt Find * format * fix merge * Shargon's feedback * rm max page number * Shargon's feedback * rename FindState to FindStates * update * FindStates ignore from * FindStates return first and last proof * fix inexistent contract * StateService: add key length check * fix * fix offset initial value Co-authored-by: Shargon Co-authored-by: Owen Zhang <38493437+superboyiii@users.noreply.github.com> --- src/StateService/MPT/Helper.cs | 18 ++++ src/StateService/MPT/MPTTrie.Delete.cs | 5 +- src/StateService/MPT/MPTTrie.Find.cs | 52 +++++++++-- src/StateService/MPT/MPTTrie.Get.cs | 12 ++- src/StateService/MPT/MPTTrie.Proof.cs | 5 +- src/StateService/README.md | 70 ++++++++++++++ src/StateService/Settings.cs | 2 + src/StateService/StatePlugin.cs | 91 ++++++++++++++++++- src/StateService/StateService/config.json | 3 +- src/StateService/Storage/StateStore.cs | 6 +- .../MPT/UT_Helper.cs | 28 ++++++ .../MPT/UT_MPTTrie.cs | 18 ++++ 12 files changed, 290 insertions(+), 20 deletions(-) create mode 100644 src/StateService/MPT/Helper.cs create mode 100644 src/StateService/README.md create mode 100644 tests/Neo.Plugins.StateService.Tests/MPT/UT_Helper.cs diff --git a/src/StateService/MPT/Helper.cs b/src/StateService/MPT/Helper.cs new file mode 100644 index 000000000..34004dd2e --- /dev/null +++ b/src/StateService/MPT/Helper.cs @@ -0,0 +1,18 @@ +using System; + +namespace Neo.Plugins.MPT +{ + public static class Helper + { + public static int CompareTo(this byte[] arr1, byte[] arr2) + { + if (arr1 is null || arr2 is null) throw new ArgumentNullException(); + for (int i = 0; i < arr1.Length && i < arr2.Length; i++) + { + var r = arr1[i].CompareTo(arr2[i]); + if (r != 0) return r; + } + return arr2.Length < arr1.Length ? 1 : arr2.Length == arr1.Length ? 0 : -1; + } + } +} diff --git a/src/StateService/MPT/MPTTrie.Delete.cs b/src/StateService/MPT/MPTTrie.Delete.cs index 55cf8d4ec..733b61c21 100644 --- a/src/StateService/MPT/MPTTrie.Delete.cs +++ b/src/StateService/MPT/MPTTrie.Delete.cs @@ -10,7 +10,10 @@ partial class MPTTrie public bool Delete(TKey key) { var path = ToNibbles(key.ToArray()); - if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length == 0) + throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length > MPTNode.MaxKeyLength) + throw new ArgumentException("exceeds limit", nameof(key)); return TryDelete(ref root, path); } diff --git a/src/StateService/MPT/MPTTrie.Find.cs b/src/StateService/MPT/MPTTrie.Find.cs index 251b6e77f..82d33f06f 100644 --- a/src/StateService/MPT/MPTTrie.Find.cs +++ b/src/StateService/MPT/MPTTrie.Find.cs @@ -62,22 +62,35 @@ private ReadOnlySpan Seek(ref MPTNode node, ReadOnlySpan path, out M return ReadOnlySpan.Empty; } - public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan prefix) + public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan prefix, byte[] from = null) { var path = ToNibbles(prefix); + int offset = 0; + if (from is null) from = Array.Empty(); + if (0 < from.Length) + { + if (!from.AsSpan().StartsWith(prefix)) + throw new InvalidOperationException("invalid from key"); + from = ToNibbles(from.AsSpan()); + } + if (path.Length > MPTNode.MaxKeyLength || from.Length > MPTNode.MaxKeyLength) + throw new ArgumentException("exceeds limit"); path = Seek(ref root, path, out MPTNode start).ToArray(); - return Travers(start, path) + offset = path.Length; + return Travers(start, path, from, offset) .Select(p => (FromNibbles(p.Key).AsSerializable(), p.Value.AsSerializable())); } - private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path) + private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path, byte[] from, int offset) { if (node is null) yield break; + if (offset < 0) throw new InvalidOperationException("invalid offset"); switch (node.Type) { case NodeType.LeafNode: { - yield return (path, (byte[])node.Value.Clone()); + if (from.Length <= offset && !path.SequenceEqual(from)) + yield return (path, (byte[])node.Value.Clone()); break; } case NodeType.Empty: @@ -87,23 +100,44 @@ private ReadOnlySpan Seek(ref MPTNode node, ReadOnlySpan path, out M var newNode = cache.Resolve(node.Hash); if (newNode is null) throw new InvalidOperationException("Internal error, can't resolve hash when mpt find"); node = newNode; - foreach (var item in Travers(node, path)) + foreach (var item in Travers(node, path, from, offset)) yield return item; break; } case NodeType.BranchNode: { - for (int i = 0; i < MPTNode.BranchChildCount; i++) + if (offset < from.Length) + { + for (int i = 0; i < MPTNode.BranchChildCount - 1; i++) + { + if (from[offset] < i) + foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, from.Length)) + yield return item; + else if (i == from[offset]) + foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset + 1)) + yield return item; + } + } + else { - foreach (var item in Travers(node.Children[i], i == MPTNode.BranchChildCount - 1 ? path : Concat(path, new byte[] { (byte)i }))) + foreach (var item in Travers(node.Children[MPTNode.BranchChildCount - 1], path, from, offset)) yield return item; + for (int i = 0; i < MPTNode.BranchChildCount - 1; i++) + { + foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset)) + yield return item; + } } break; } case NodeType.ExtensionNode: { - foreach (var item in Travers(node.Next, Concat(path, node.Key))) - yield return item; + if (offset < from.Length && from.AsSpan()[offset..].StartsWith(node.Key)) + foreach (var item in Travers(node.Next, Concat(path, node.Key), from, offset + node.Key.Length)) + yield return item; + else if (from.Length <= offset || 0 < node.Key.CompareTo(from[offset..])) + foreach (var item in Travers(node.Next, Concat(path, node.Key), from, from.Length)) + yield return item; break; } } diff --git a/src/StateService/MPT/MPTTrie.Get.cs b/src/StateService/MPT/MPTTrie.Get.cs index 335a44773..6887e5e3a 100644 --- a/src/StateService/MPT/MPTTrie.Get.cs +++ b/src/StateService/MPT/MPTTrie.Get.cs @@ -11,7 +11,10 @@ public TValue this[TKey key] get { var path = ToNibbles(key.ToArray()); - if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length == 0) + throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length > MPTNode.MaxKeyLength) + throw new ArgumentException("exceeds limit", nameof(key)); var result = TryGet(ref root, path, out var value); return result ? value.ToArray().AsSerializable() : throw new KeyNotFoundException(); } @@ -21,10 +24,13 @@ public bool TryGetValue(TKey key, out TValue value) { value = default; var path = ToNibbles(key.ToArray()); - if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length == 0) + throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length > MPTNode.MaxKeyLength) + throw new ArgumentException("exceeds limit", nameof(key)); var result = TryGet(ref root, path, out var val); if (result) - val.ToArray().AsSerializable(); + value = val.ToArray().AsSerializable(); return result; } diff --git a/src/StateService/MPT/MPTTrie.Proof.cs b/src/StateService/MPT/MPTTrie.Proof.cs index 42999d250..6b178a027 100644 --- a/src/StateService/MPT/MPTTrie.Proof.cs +++ b/src/StateService/MPT/MPTTrie.Proof.cs @@ -13,7 +13,10 @@ partial class MPTTrie public bool TryGetProof(TKey key, out HashSet proof) { var path = ToNibbles(key.ToArray()); - if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length == 0) + throw new ArgumentException("could not be empty", nameof(key)); + if (path.Length > MPTNode.MaxKeyLength) + throw new ArgumentException("exceeds limit", nameof(key)); proof = new HashSet(ByteArrayEqualityComparer.Default); return GetProof(ref root, path, proof); } diff --git a/src/StateService/README.md b/src/StateService/README.md new file mode 100644 index 000000000..c8e6a2ef0 --- /dev/null +++ b/src/StateService/README.md @@ -0,0 +1,70 @@ +# StateService + +## RPC API + +### GetStateRoot +#### Params +|Name|Type|Summary|Required| +|-|-|-|-| +|Index|uint|index|true| +#### Result +StateRoot Object +|Name|Type|Summary| +|-|-|-| +|version|number|version| +|index|number|index| +|roothash|string|version| +|witness|Object|witness from validators| + +### GetProof +#### Params +|Name|Type|Summary|Required| +|-|-|-|-| +|RootHash|UInt256|state root|true| +|ScriptHash|UInt160|contract script hash|true| +|Key|base64 string|key|true| +#### Result +Proof in base64 string + +### VerifyProof +#### Params +|Name|Type|Summary| +|-|-|-| +|RootHash|UInt256|state root|true| +|Proof|base64 string|proof|true| +#### Result +Value in base64 string + +### GetStateheight +#### Result +|Name|Type|Summary| +|-|-|-| +|localrootindex|number|root hash index calculated locally| +|validatedrootindex|number|root hash index verified by validators| + +### GetState +#### Params +|Name|Type|Summary|Required| +|-|-|-|-| +|RootHash|UInt256|specify state|true| +|ScriptHash|UInt160|contract script hash|true| +|Key|base64 string|key|true| +#### Result +Value in base64 string or `null` + +### FindStates +#### Params +|Name|Type|Summary|Required| +|-|-|-|-| +|RootHash|UInt256|specify state|true| +|ScriptHash|UInt160|contract script hash|true| +|Prefix|base64 string|key prefix|true| +|From|base64 string|start key, default `Empty`|optional| +|Count|number|count of results in one request, default `MaxFindResultItems`|optional| +#### Result +|Name|Type|Summary| +|-|-|-| +|firstProof|string|proof of first value in results| +|lastProof|string|proof of last value in results| +|truncated|bool|whether the results is truncated because of limitation| +|results|array|key-values found| diff --git a/src/StateService/Settings.cs b/src/StateService/Settings.cs index 2c7dcf97e..3696d3a67 100644 --- a/src/StateService/Settings.cs +++ b/src/StateService/Settings.cs @@ -8,6 +8,7 @@ internal class Settings public bool FullState { get; } public uint Network { get; } public bool AutoVerify { get; } + public int MaxFindResultItems { get; } public static Settings Default { get; private set; } @@ -17,6 +18,7 @@ private Settings(IConfigurationSection section) FullState = section.GetValue("FullState", false); Network = section.GetValue("Network", 5195086u); AutoVerify = section.GetValue("AutoVerify", false); + MaxFindResultItems = section.GetValue("MaxFindResultItems", 100); } public static void Load(IConfigurationSection section) diff --git a/src/StateService/StatePlugin.cs b/src/StateService/StatePlugin.cs index 0fe1988d5..5d8cb3c96 100644 --- a/src/StateService/StatePlugin.cs +++ b/src/StateService/StatePlugin.cs @@ -177,7 +177,10 @@ private string GetProof(UInt256 root_hash, UInt160 script_hash, byte[] key) Id = contract.Id, Key = key, }; - var result = StateStore.Singleton.TryGetProof(root_hash, skey, out var proof); + + using ISnapshot store = StateStore.Singleton.GetStoreSnapshot(); + var trie = new MPTTrie(store, root_hash); + var result = trie.TryGetProof(skey, out var proof); if (!result) throw new RpcException(-100, "Unknown value"); using MemoryStream ms = new MemoryStream(); @@ -239,5 +242,91 @@ public JObject GetStateHeight(JArray _params) json["validatedrootindex"] = StateStore.Singleton.ValidatedRootIndex; return json; } + + private ContractState GetHistoricalContractState(MPTTrie trie, UInt160 script_hash) + { + const byte prefix = 8; + StorageKey skey = new KeyBuilder(NativeContract.ContractManagement.Id, prefix).Add(script_hash); + return trie.TryGetValue(skey, out var value) ? value.GetInteroperable() : null; + } + + [RpcMethod] + public JObject FindStates(JArray _params) + { + var root_hash = UInt256.Parse(_params[0].AsString()); + if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash) + throw new RpcException(-100, "Old state not supported"); + var script_hash = UInt160.Parse(_params[1].AsString()); + var prefix = Convert.FromBase64String(_params[2].AsString()); + byte[] key = Array.Empty(); + if (3 < _params.Count) + key = Convert.FromBase64String(_params[3].AsString()); + int count = Settings.Default.MaxFindResultItems; + if (4 < _params.Count) + count = int.Parse(_params[4].AsString()); + if (Settings.Default.MaxFindResultItems < count) + count = Settings.Default.MaxFindResultItems; + using var store = StateStore.Singleton.GetStoreSnapshot(); + var trie = new MPTTrie(store, root_hash); + var contract = GetHistoricalContractState(trie, script_hash); + if (contract is null) throw new RpcException(-100, "Unknown contract"); + StorageKey pkey = new() + { + Id = contract.Id, + Key = prefix, + }; + StorageKey fkey = new() + { + Id = pkey.Id, + Key = key, + }; + JObject json = new(); + JArray jarr = new(); + int i = 0; + foreach (var (ikey, ivalue) in trie.Find(pkey.ToArray(), 0 < key.Length ? fkey.ToArray() : null)) + { + if (count < i) break; + if (i < count) + { + JObject j = new(); + j["key"] = Convert.ToBase64String(ikey.Key); + j["value"] = Convert.ToBase64String(ivalue.Value); + jarr.Add(j); + } + i++; + }; + if (0 < jarr.Count) + { + json["firstProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.First()["key"].AsString())); + } + if (1 < jarr.Count) + { + json["lastProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.Last()["key"].AsString())); + } + json["truncated"] = count < i; + json["results"] = jarr; + return json; + } + + [RpcMethod] + public JObject GetState(JArray _params) + { + var root_hash = UInt256.Parse(_params[0].AsString()); + if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash) + throw new RpcException(-100, "Old state not supported"); + var script_hash = UInt160.Parse(_params[1].AsString()); + var key = Convert.FromBase64String(_params[2].AsString()); + using var store = StateStore.Singleton.GetStoreSnapshot(); + var trie = new MPTTrie(store, root_hash); + + var contract = GetHistoricalContractState(trie, script_hash); + if (contract is null) throw new RpcException(-100, "Unknown contract"); + StorageKey skey = new() + { + Id = contract.Id, + Key = key, + }; + return Convert.ToBase64String(trie[skey].Value); + } } } diff --git a/src/StateService/StateService/config.json b/src/StateService/StateService/config.json index e1f097ab6..265436fc3 100644 --- a/src/StateService/StateService/config.json +++ b/src/StateService/StateService/config.json @@ -3,7 +3,8 @@ "Path": "Data_MPT_{0}", "FullState": false, "Network": 860833102, - "AutoVerify": false + "AutoVerify": false, + "MaxFindResultItems": 100 }, "Dependency": [ "RpcServer" diff --git a/src/StateService/Storage/StateStore.cs b/src/StateService/Storage/StateStore.cs index 3336816a0..c8c6dc9e7 100644 --- a/src/StateService/Storage/StateStore.cs +++ b/src/StateService/Storage/StateStore.cs @@ -55,11 +55,9 @@ public StateSnapshot GetSnapshot() return new StateSnapshot(store); } - public bool TryGetProof(UInt256 root, StorageKey skey, out HashSet proof) + public ISnapshot GetStoreSnapshot() { - using ISnapshot snapshot = store.GetSnapshot(); - var trie = new MPTTrie(snapshot, root); - return trie.TryGetProof(skey, out proof); + return store.GetSnapshot(); } protected override void OnReceive(object message) diff --git a/tests/Neo.Plugins.StateService.Tests/MPT/UT_Helper.cs b/tests/Neo.Plugins.StateService.Tests/MPT/UT_Helper.cs new file mode 100644 index 000000000..7d31ea3a9 --- /dev/null +++ b/tests/Neo.Plugins.StateService.Tests/MPT/UT_Helper.cs @@ -0,0 +1,28 @@ +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using static Neo.Plugins.MPT.Helper; + +namespace Neo.Plugins.StateService.Tests +{ + [TestClass] + public class UT_Helper + { + [TestMethod] + public void TestCompareTo() + { + var arr1 = new byte[] { 0, 1, 2 }; + var arr2 = new byte[] { 0, 1, 2 }; + Assert.AreEqual(0, arr1.CompareTo(arr2)); + arr1 = new byte[] { 0, 1 }; + Assert.AreEqual(-1, arr1.CompareTo(arr2)); + arr2 = new byte[] { 0 }; + Assert.AreEqual(1, arr1.CompareTo(arr2)); + arr2 = new byte[] { 0, 2 }; + Assert.AreEqual(-1, arr1.CompareTo(arr2)); + arr1 = new byte[] { 0, 3, 1 }; + Assert.AreEqual(1, arr1.CompareTo(arr2)); + Assert.AreEqual(0, Array.Empty().CompareTo(Array.Empty())); + Assert.ThrowsException(() => arr1.CompareTo(null)); + } + } +} diff --git a/tests/Neo.Plugins.StateService.Tests/MPT/UT_MPTTrie.cs b/tests/Neo.Plugins.StateService.Tests/MPT/UT_MPTTrie.cs index 5598b9d3e..9dd341e14 100644 --- a/tests/Neo.Plugins.StateService.Tests/MPT/UT_MPTTrie.cs +++ b/tests/Neo.Plugins.StateService.Tests/MPT/UT_MPTTrie.cs @@ -601,5 +601,23 @@ public void TestEmptyValueIssue633() Assert.IsNotNull(val); Assert.AreEqual(0, val.Size); } + + [TestMethod] + public void TestFindWithFrom() + { + var snapshot = new TestSnapshot(); + var mpt = new MPTTrie(snapshot, null); + mpt.Put("aa".HexToBytes(), "02".HexToBytes()); + mpt.Put("aa10".HexToBytes(), "03".HexToBytes()); + mpt.Put("aa50".HexToBytes(), "04".HexToBytes()); + var r = mpt.Find("aa".HexToBytes()).ToList(); + Assert.AreEqual(3, r.Count); + r = mpt.Find("aa".HexToBytes(), "aa30".HexToBytes()).ToList(); + Assert.AreEqual(1, r.Count); + r = mpt.Find("aa".HexToBytes(), "aa60".HexToBytes()).ToList(); + Assert.AreEqual(0, r.Count); + r = mpt.Find("aa".HexToBytes(), "aa10".HexToBytes()).ToList(); + Assert.AreEqual(1, r.Count);//without from key + } } }