[StateService] get historical state (#638)

* StateService: add historical state * GetState and FindState; Results in pages * use root hash * StateService: add from in mpt Find * format * fix merge * Shargon's feedback * rm max page number * Shargon's feedback * rename FindState to FindStates * update * FindStates ignore from * FindStates return first and last proof * fix inexistent contract * StateService: add key length check * fix * fix offset initial value Co-authored-by: Shargon <[email protected]> Co-authored-by: Owen Zhang <[email protected]>
neo-project · Sep 29, 2021 · 50f0473 · 50f0473
1 parent 87ed9f0
commit 50f0473
Show file tree

Hide file tree

Showing 12 changed files with 290 additions and 20 deletions.
diff --git a/src/StateService/MPT/Helper.cs b/src/StateService/MPT/Helper.cs
@@ -0,0 +1,18 @@
+using System;
+
+namespace Neo.Plugins.MPT
+{
+    public static class Helper
+    {
+        public static int CompareTo(this byte[] arr1, byte[] arr2)
+        {
+            if (arr1 is null || arr2 is null) throw new ArgumentNullException();
+            for (int i = 0; i < arr1.Length && i < arr2.Length; i++)
+            {
+                var r = arr1[i].CompareTo(arr2[i]);
+                if (r != 0) return r;
+            }
+            return arr2.Length < arr1.Length ? 1 : arr2.Length == arr1.Length ? 0 : -1;
+        }
+    }
+}
diff --git a/src/StateService/MPT/MPTTrie.Delete.cs b/src/StateService/MPT/MPTTrie.Delete.cs
@@ -10,7 +10,10 @@ partial class MPTTrie<TKey, TValue>
         public bool Delete(TKey key)
         {
             var path = ToNibbles(key.ToArray());
-            if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length == 0)
+                throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length > MPTNode.MaxKeyLength)
+                throw new ArgumentException("exceeds limit", nameof(key));
             return TryDelete(ref root, path);
         }
 

diff --git a/src/StateService/MPT/MPTTrie.Find.cs b/src/StateService/MPT/MPTTrie.Find.cs
@@ -62,22 +62,35 @@ private ReadOnlySpan<byte> Seek(ref MPTNode node, ReadOnlySpan<byte> path, out M
             return ReadOnlySpan<byte>.Empty;
         }
 
-        public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan<byte> prefix)
+        public IEnumerable<(TKey Key, TValue Value)> Find(ReadOnlySpan<byte> prefix, byte[] from = null)
         {
             var path = ToNibbles(prefix);
+            int offset = 0;
+            if (from is null) from = Array.Empty<byte>();
+            if (0 < from.Length)
+            {
+                if (!from.AsSpan().StartsWith(prefix))
+                    throw new InvalidOperationException("invalid from key");
+                from = ToNibbles(from.AsSpan());
+            }
+            if (path.Length > MPTNode.MaxKeyLength || from.Length > MPTNode.MaxKeyLength)
+                throw new ArgumentException("exceeds limit");
             path = Seek(ref root, path, out MPTNode start).ToArray();
-            return Travers(start, path)
+            offset = path.Length;
+            return Travers(start, path, from, offset)
                 .Select(p => (FromNibbles(p.Key).AsSerializable<TKey>(), p.Value.AsSerializable<TValue>()));
         }
 
-        private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path)
+        private IEnumerable<(byte[] Key, byte[] Value)> Travers(MPTNode node, byte[] path, byte[] from, int offset)
         {
             if (node is null) yield break;
+            if (offset < 0) throw new InvalidOperationException("invalid offset");
             switch (node.Type)
             {
                 case NodeType.LeafNode:
                     {
-                        yield return (path, (byte[])node.Value.Clone());
+                        if (from.Length <= offset && !path.SequenceEqual(from))
+                            yield return (path, (byte[])node.Value.Clone());
                         break;
                     }
                 case NodeType.Empty:
@@ -87,23 +100,44 @@ private ReadOnlySpan<byte> Seek(ref MPTNode node, ReadOnlySpan<byte> path, out M
                         var newNode = cache.Resolve(node.Hash);
                         if (newNode is null) throw new InvalidOperationException("Internal error, can't resolve hash when mpt find");
                         node = newNode;
-                        foreach (var item in Travers(node, path))
+                        foreach (var item in Travers(node, path, from, offset))
                             yield return item;
                         break;
                     }
                 case NodeType.BranchNode:
                     {
-                        for (int i = 0; i < MPTNode.BranchChildCount; i++)
+                        if (offset < from.Length)
+                        {
+                            for (int i = 0; i < MPTNode.BranchChildCount - 1; i++)
+                            {
+                                if (from[offset] < i)
+                                    foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, from.Length))
+                                        yield return item;
+                                else if (i == from[offset])
+                                    foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset + 1))
+                                        yield return item;
+                            }
+                        }
+                        else
                         {
-                            foreach (var item in Travers(node.Children[i], i == MPTNode.BranchChildCount - 1 ? path : Concat(path, new byte[] { (byte)i })))
+                            foreach (var item in Travers(node.Children[MPTNode.BranchChildCount - 1], path, from, offset))
                                 yield return item;
+                            for (int i = 0; i < MPTNode.BranchChildCount - 1; i++)
+                            {
+                                foreach (var item in Travers(node.Children[i], Concat(path, new byte[] { (byte)i }), from, offset))
+                                    yield return item;
+                            }
                         }
                         break;
                     }
                 case NodeType.ExtensionNode:
                     {
-                        foreach (var item in Travers(node.Next, Concat(path, node.Key)))
-                            yield return item;
+                        if (offset < from.Length && from.AsSpan()[offset..].StartsWith(node.Key))
+                            foreach (var item in Travers(node.Next, Concat(path, node.Key), from, offset + node.Key.Length))
+                                yield return item;
+                        else if (from.Length <= offset || 0 < node.Key.CompareTo(from[offset..]))
+                            foreach (var item in Travers(node.Next, Concat(path, node.Key), from, from.Length))
+                                yield return item;
                         break;
                     }
             }

diff --git a/src/StateService/MPT/MPTTrie.Get.cs b/src/StateService/MPT/MPTTrie.Get.cs
@@ -11,7 +11,10 @@ public TValue this[TKey key]
             get
             {
                 var path = ToNibbles(key.ToArray());
-                if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key));
+                if (path.Length == 0)
+                    throw new ArgumentException("could not be empty", nameof(key));
+                if (path.Length > MPTNode.MaxKeyLength)
+                    throw new ArgumentException("exceeds limit", nameof(key));
                 var result = TryGet(ref root, path, out var value);
                 return result ? value.ToArray().AsSerializable<TValue>() : throw new KeyNotFoundException();
             }
@@ -21,10 +24,13 @@ public bool TryGetValue(TKey key, out TValue value)
         {
             value = default;
             var path = ToNibbles(key.ToArray());
-            if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length == 0)
+                throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length > MPTNode.MaxKeyLength)
+                throw new ArgumentException("exceeds limit", nameof(key));
             var result = TryGet(ref root, path, out var val);
             if (result)
-                val.ToArray().AsSerializable<TValue>();
+                value = val.ToArray().AsSerializable<TValue>();
             return result;
         }
 

diff --git a/src/StateService/MPT/MPTTrie.Proof.cs b/src/StateService/MPT/MPTTrie.Proof.cs
@@ -13,7 +13,10 @@ partial class MPTTrie<TKey, TValue>
         public bool TryGetProof(TKey key, out HashSet<byte[]> proof)
         {
             var path = ToNibbles(key.ToArray());
-            if (path.Length == 0) throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length == 0)
+                throw new ArgumentException("could not be empty", nameof(key));
+            if (path.Length > MPTNode.MaxKeyLength)
+                throw new ArgumentException("exceeds limit", nameof(key));
             proof = new HashSet<byte[]>(ByteArrayEqualityComparer.Default);
             return GetProof(ref root, path, proof);
         }

diff --git a/src/StateService/README.md b/src/StateService/README.md
@@ -0,0 +1,70 @@
+# StateService
+
+## RPC API
+
+### GetStateRoot
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|Index|uint|index|true|
+#### Result
+StateRoot Object
+|Name|Type|Summary|
+|-|-|-|
+|version|number|version|
+|index|number|index|
+|roothash|string|version|
+|witness|Object|witness from validators|
+
+### GetProof
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|state root|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Key|base64 string|key|true|
+#### Result
+Proof in base64 string
+
+### VerifyProof
+#### Params
+|Name|Type|Summary|
+|-|-|-|
+|RootHash|UInt256|state root|true|
+|Proof|base64 string|proof|true|
+#### Result
+Value in base64 string
+
+### GetStateheight
+#### Result
+|Name|Type|Summary|
+|-|-|-|
+|localrootindex|number|root hash index calculated locally|
+|validatedrootindex|number|root hash index verified by validators|
+
+### GetState
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|specify state|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Key|base64 string|key|true|
+#### Result
+Value in base64 string or `null`
+
+### FindStates
+#### Params
+|Name|Type|Summary|Required|
+|-|-|-|-|
+|RootHash|UInt256|specify state|true|
+|ScriptHash|UInt160|contract script hash|true|
+|Prefix|base64 string|key prefix|true|
+|From|base64 string|start key, default `Empty`|optional|
+|Count|number|count of results in one request, default `MaxFindResultItems`|optional|
+#### Result
+|Name|Type|Summary|
+|-|-|-|
+|firstProof|string|proof of first value in results|
+|lastProof|string|proof of last value in results|
+|truncated|bool|whether the results is truncated because of limitation|
+|results|array|key-values found|
diff --git a/src/StateService/Settings.cs b/src/StateService/Settings.cs
@@ -8,6 +8,7 @@ internal class Settings
         public bool FullState { get; }
         public uint Network { get; }
         public bool AutoVerify { get; }
+        public int MaxFindResultItems { get; }
 
         public static Settings Default { get; private set; }
 
@@ -17,6 +18,7 @@ private Settings(IConfigurationSection section)
             FullState = section.GetValue("FullState", false);
             Network = section.GetValue("Network", 5195086u);
             AutoVerify = section.GetValue("AutoVerify", false);
+            MaxFindResultItems = section.GetValue("MaxFindResultItems", 100);
         }
 
         public static void Load(IConfigurationSection section)

diff --git a/src/StateService/StatePlugin.cs b/src/StateService/StatePlugin.cs
@@ -177,7 +177,10 @@ private string GetProof(UInt256 root_hash, UInt160 script_hash, byte[] key)
                 Id = contract.Id,
                 Key = key,
             };
-            var result = StateStore.Singleton.TryGetProof(root_hash, skey, out var proof);
+
+            using ISnapshot store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+            var result = trie.TryGetProof(skey, out var proof);
             if (!result) throw new RpcException(-100, "Unknown value");
 
             using MemoryStream ms = new MemoryStream();
@@ -239,5 +242,91 @@ public JObject GetStateHeight(JArray _params)
             json["validatedrootindex"] = StateStore.Singleton.ValidatedRootIndex;
             return json;
         }
+
+        private ContractState GetHistoricalContractState(MPTTrie<StorageKey, StorageItem> trie, UInt160 script_hash)
+        {
+            const byte prefix = 8;
+            StorageKey skey = new KeyBuilder(NativeContract.ContractManagement.Id, prefix).Add(script_hash);
+            return trie.TryGetValue(skey, out var value) ? value.GetInteroperable<ContractState>() : null;
+        }
+
+        [RpcMethod]
+        public JObject FindStates(JArray _params)
+        {
+            var root_hash = UInt256.Parse(_params[0].AsString());
+            if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash)
+                throw new RpcException(-100, "Old state not supported");
+            var script_hash = UInt160.Parse(_params[1].AsString());
+            var prefix = Convert.FromBase64String(_params[2].AsString());
+            byte[] key = Array.Empty<byte>();
+            if (3 < _params.Count)
+                key = Convert.FromBase64String(_params[3].AsString());
+            int count = Settings.Default.MaxFindResultItems;
+            if (4 < _params.Count)
+                count = int.Parse(_params[4].AsString());
+            if (Settings.Default.MaxFindResultItems < count)
+                count = Settings.Default.MaxFindResultItems;
+            using var store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+            var contract = GetHistoricalContractState(trie, script_hash);
+            if (contract is null) throw new RpcException(-100, "Unknown contract");
+            StorageKey pkey = new()
+            {
+                Id = contract.Id,
+                Key = prefix,
+            };
+            StorageKey fkey = new()
+            {
+                Id = pkey.Id,
+                Key = key,
+            };
+            JObject json = new();
+            JArray jarr = new();
+            int i = 0;
+            foreach (var (ikey, ivalue) in trie.Find(pkey.ToArray(), 0 < key.Length ? fkey.ToArray() : null))
+            {
+                if (count < i) break;
+                if (i < count)
+                {
+                    JObject j = new();
+                    j["key"] = Convert.ToBase64String(ikey.Key);
+                    j["value"] = Convert.ToBase64String(ivalue.Value);
+                    jarr.Add(j);
+                }
+                i++;
+            };
+            if (0 < jarr.Count)
+            {
+                json["firstProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.First()["key"].AsString()));
+            }
+            if (1 < jarr.Count)
+            {
+                json["lastProof"] = GetProof(root_hash, script_hash, Convert.FromBase64String(jarr.Last()["key"].AsString()));
+            }
+            json["truncated"] = count < i;
+            json["results"] = jarr;
+            return json;
+        }
+
+        [RpcMethod]
+        public JObject GetState(JArray _params)
+        {
+            var root_hash = UInt256.Parse(_params[0].AsString());
+            if (!Settings.Default.FullState && StateStore.Singleton.CurrentLocalRootHash != root_hash)
+                throw new RpcException(-100, "Old state not supported");
+            var script_hash = UInt160.Parse(_params[1].AsString());
+            var key = Convert.FromBase64String(_params[2].AsString());
+            using var store = StateStore.Singleton.GetStoreSnapshot();
+            var trie = new MPTTrie<StorageKey, StorageItem>(store, root_hash);
+
+            var contract = GetHistoricalContractState(trie, script_hash);
+            if (contract is null) throw new RpcException(-100, "Unknown contract");
+            StorageKey skey = new()
+            {
+                Id = contract.Id,
+                Key = key,
+            };
+            return Convert.ToBase64String(trie[skey].Value);
+        }
     }
 }
diff --git a/src/StateService/StateService/config.json b/src/StateService/StateService/config.json
@@ -3,7 +3,8 @@
     "Path": "Data_MPT_{0}",
     "FullState": false,
     "Network": 860833102,
-    "AutoVerify": false
+    "AutoVerify": false,
+    "MaxFindResultItems": 100
   },
   "Dependency": [
     "RpcServer"

diff --git a/src/StateService/Storage/StateStore.cs b/src/StateService/Storage/StateStore.cs
@@ -55,11 +55,9 @@ public StateSnapshot GetSnapshot()
             return new StateSnapshot(store);
         }
 
-        public bool TryGetProof(UInt256 root, StorageKey skey, out HashSet<byte[]> proof)
+        public ISnapshot GetStoreSnapshot()
         {
-            using ISnapshot snapshot = store.GetSnapshot();
-            var trie = new MPTTrie<StorageKey, StorageItem>(snapshot, root);
-            return trie.TryGetProof(skey, out proof);
+            return store.GetSnapshot();
         }
 
         protected override void OnReceive(object message)